diff --git a/.bazelrc b/.bazelrc index 308ab55ec..b700f8961 100644 --- a/.bazelrc +++ b/.bazelrc @@ -99,6 +99,12 @@ test:embedded --test_timeout=-1,-1,-1,1200 test:embedded --test_arg=-test.parallel=4 test:embedded --test_env=GO_TEST_WRAP_TESTV=1 test:embedded --remote_download_regex=.*/test\.xml$ +# Since D2 step 1 this lane is the tier's only pre-merge run on same-repo PRs +# (pr-risk.yml's legacy jobs stand down), and the legacy jobs ran every test +# every time (-test.count=1). A cached PASS would let a stale or poisoned +# farm action-cache entry, or an unreproduced -race finding, stand in for a +# run: always execute, at farm (not runner) cost. +test:embedded --nocache_test_results # Dolt-server lane: the same jobs as the docker lane ("Test (storage domain + # uow)" and "Contract corpus"), but every dolt sql-server is a child of the diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index a587814c4..265aa2bad 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -5,7 +5,13 @@ # and server-Dolt storage tiers, main.yml's integration jobs). # # PRs and merge groups run it through pr.yml's `bazel` job (workflow_call), -# in the same run as the legacy jobs it mirrors, which stay required too. +# in the same run as the legacy jobs it mirrors, which stay required too, +# except pr-risk.yml's embedded-Dolt test jobs: while the committed +# BAZEL_RETIRES_LEGACY_EMBEDDED flag is "true", on a same-repo, +# non-Dependabot pull_request (whatever the mode), pr-risk.yml's bazel-embedded-coverage job stands them down and +# bazel-embedded alone runs the tier; pr.yml's ci-gate then requires that +# lane to have run remotely and passed, so modes skip and local are red there +# (scripts/pr_risk_bazel_coverage_test.go). # This file has no pull_request or merge_group trigger of its own, so a PR # runs the lane once. Every lane reports its job.status as a workflow_call # output (named after the job) from an always() last step. pr.yml's ci-gate @@ -659,6 +665,9 @@ jobs: # processes (~7-10k test-seconds) on one runner, an hour or more per run # on the pool whose job cap this track exists to relieve, and it proves # nothing pr-risk.yml's embedded jobs (which do run for forks) do not. + # On same-repo pull_requests those jobs stand down (pr-risk.yml's + # bazel-embedded-coverage), so this lane is then the tier's only PR run + # and pr.yml's gate requires it to have run here and passed. # The runs-on expression is the shared one and always picks the # Blacksmith runner under this if. needs: rbe @@ -687,6 +696,18 @@ jobs: RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + # Bazel retries a flaky = True test (up to three attempts) and reports a + # later pass as PASSED. The policy test greps BUILD files for it; this + # asks Bazel itself, so a macro or a variable cannot hide one. Cheap: + # loading only, with the repository cache the test step needs anyway. + - name: No test target is marked flaky + run: | + flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" + if [ -n "$flaky" ]; then + echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" + exit 1 + fi + # --build_event_json_file is a command option, so it goes after `test`. - name: bazel test //... --config=embedded id: test @@ -708,6 +729,24 @@ jobs: if: ${{ always() && steps.test.outcome != 'skipped' }} run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + # Since D2 step 1 this lane is the tier's only pre-merge run on + # same-repo PRs. The shard scripts discover tests from source, but each + # Bazel binary holds only its target's srcs: a listed test the binary + # lacks matches nothing and passes silently. Fail unless every Bazel + # shard ran exactly the tests its script lists (list-only mode, cheap), + # and unless every shard and conformance partition ran at least one + # top-level test that was not skipped (an injected -test.short or + # BEADS_TEST_SKIP turns the tier into t.Skip calls). + - name: Every listed test ran in its shard + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: >- + python3 tools/bazel/check_shard_coverage.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 20 + --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 5 + --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test + --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test + - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 3c293aff9..386fa6575 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -12,6 +12,15 @@ concurrency: env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" + # D2 step 1: "true" retires PR Risk's legacy embedded-Dolt test jobs on + # same-repo, non-Dependabot pull_requests; pr.yml's + # gated Bazel lane is then the tier's only pre-merge run, and pr.yml's + # gate goes red unless that lane ran remotely and passed. Committed, not a + # repo variable, so a re-run can never change it; must equal pr.yml's / + # pr-risk.yml's copy (scripts/pr_risk_bazel_coverage_test.go). Revert: + # commit "false" in both workflows (in-flight PRs pick it up on their next + # push or merge of main). + BAZEL_RETIRES_LEGACY_EMBEDDED: "true" jobs: detect-ci-tier: @@ -55,6 +64,52 @@ jobs: printf '%s\n' "$selection" | sed 's/^/- `/' | sed 's/$/`/' } >> "$GITHUB_STEP_SUMMARY" + # D2 step 1: whether the legacy test-embedded-* jobs below stand down + # because pr.yml's gated Bazel `embedded Dolt tier` lane is this PR's run of + # the tier. True only with BAZEL_RETIRES_LEGACY_EMBEDDED, on a same-repo (not + # fork) pull_request whose actor is not Dependabot. pr.yml runs this same + # job (policy-tested identical) and its gate goes red unless the lane then + # ran remotely and passed, so neither the kill switch (mode skip) nor a + # missing executor secret (mode local) can silently drop the tier. Forks + # and Dependabot (no secrets), merge_group and every other event keep the + # legacy tier. The step reads no secret or variable and runs no repository + # code. + bazel-embedded-coverage: + name: Bazel embedded coverage + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + covered: ${{ steps.decide.outputs.covered }} + steps: + # GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's + # expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS + # and not the executor secret (either, evaluated per run, would let one + # workflow's re-run disagree with the other's). A same-repo PR whose run + # lacks the secret is covered anyway, so pr.yml's gate goes red (mode + # local) instead of the tier silently moving back. Dependabot runs get + # no Actions secrets by design and keep the legacy tier; github.actor + # (unlike triggering_actor) stays the PR's actor on a re-run. + - name: Decide whether the Bazel lane covers the embedded tier + id: decide + env: + RETIRED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} + PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + FORK: ${{ github.event.pull_request.head.repo.fork == true }} + DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} + run: | + for v in RETIRED PULL_REQUEST FORK DEPENDABOT; do + case "${!v}" in + true|false) ;; + *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; + esac + done + covered=false + if [ "$RETIRED" = true ] && [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then + covered=true + fi + echo "covered=$covered" >> "$GITHUB_OUTPUT" + echo "Legacy embedded Dolt tier retired in favour of the Bazel lane: **$covered** (retired: $RETIRED, pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT)" | tee -a "$GITHUB_STEP_SUMMARY" + # End-to-end coverage for proxied-server CLI subcommands, sharded across the # CI matrix. Each sub-test drives the bd CLI subprocess against a real dolt # sql-server started via testcontainers by the shared proxied harness @@ -119,6 +174,11 @@ jobs: # Split into: build → storage tests + parallel cmd test shards. # cmd/bd embedded tests use a committed shard manifest; newly-added tests # still auto-distribute via hash fallback until the manifest is updated. + # + # The storage, conformance and cmd test jobs stand down when + # bazel-embedded-coverage says pr.yml's gated Bazel `embedded Dolt tier` + # lane runs them instead; build-embedded keeps running, because its + # artifact also feeds the proxied and server Dolt jobs. build-embedded: name: Build (Embedded Dolt) @@ -202,8 +262,8 @@ jobs: test-embedded-storage: name: Test (Embedded Dolt Storage ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' + needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -240,8 +300,8 @@ jobs: test-embedded-conformance: name: Test (Embedded Dolt Conformance - ${{ matrix.partition }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' + needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' runs-on: ubuntu-latest timeout-minutes: 35 strategy: @@ -391,8 +451,8 @@ jobs: test-embedded-cmd: name: Test (Embedded Dolt Cmd ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' + needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' runs-on: ubuntu-latest strategy: fail-fast: false @@ -457,6 +517,7 @@ jobs: runs-on: ubuntu-latest needs: - detect-ci-tier + - bazel-embedded-coverage - build-embedded - test-embedded-storage - test-embedded-conformance @@ -474,6 +535,7 @@ jobs: CI_GATE_NAME: PR risk gate CI_GATE_REQUIRED: >- DETECT_CI_TIER + BAZEL_EMBEDDED_COVERAGE BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE @@ -483,7 +545,9 @@ jobs: TEST_PROXIED_CMD TEST_NIX FULL_EMBEDDED: ${{ needs.detect-ci-tier.outputs.full_embedded }} + BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-embedded-coverage.outputs.covered }} DETECT_CI_TIER: ${{ needs.detect-ci-tier.result }} + BAZEL_EMBEDDED_COVERAGE: ${{ needs.bazel-embedded-coverage.result }} BUILD_EMBEDDED: ${{ needs.build-embedded.result }} TEST_EMBEDDED_STORAGE: ${{ needs.test-embedded-storage.result }} TEST_EMBEDDED_CONFORMANCE: ${{ needs.test-embedded-conformance.result }} @@ -497,5 +561,11 @@ jobs: if [[ "$FULL_EMBEDDED" != "true" ]]; then skipped_ok="BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD" fi + # The legacy embedded test jobs stand down only where pr.yml's gated + # Bazel lane runs the tier (bazel-embedded-coverage); any other skip + # of them, or a failed or missing decision, fails the gate. + if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]]; then + skipped_ok="$skipped_ok TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_EMBEDDED_CMD" + fi export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b782fba3f..e45c3a90a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -15,6 +15,15 @@ concurrency: env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" + # D2 step 1: "true" retires PR Risk's legacy embedded-Dolt test jobs on + # same-repo, non-Dependabot pull_requests; pr.yml's + # gated Bazel lane is then the tier's only pre-merge run, and pr.yml's + # gate goes red unless that lane ran remotely and passed. Committed, not a + # repo variable, so a re-run can never change it; must equal pr.yml's / + # pr-risk.yml's copy (scripts/pr_risk_bazel_coverage_test.go). Revert: + # commit "false" in both workflows (in-flight PRs pick it up on their next + # push or merge of main). + BAZEL_RETIRES_LEGACY_EMBEDDED: "true" jobs: @@ -1760,7 +1769,11 @@ jobs: # The Bazel lane (bazel.yml), gated alongside the legacy jobs it mirrors # (PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and # contract corpus, and pr-risk.yml's embedded-Dolt, proxied-server and - # server-Dolt storage tiers), all of which stay required. bazel.yml has no pull_request/merge_group trigger of its own: + # server-Dolt storage tiers), all of which stay required, except that + # pr-risk.yml's embedded-Dolt test jobs stand down on same-repo PRs with + # a non-Dependabot actor while BAZEL_RETIRES_LEGACY_EMBEDDED is "true" (its + # bazel-embedded-coverage job; D2 step 1), and ci-gate then requires this + # call's embedded lane to have run remotely and passed. bazel.yml has no pull_request/merge_group trigger of its own: # this call is its only PR run. It gets the four RBE secrets and nothing # else; its rbe job decides remote, local (forks, Dependabot: no # remote-only lanes) or skip, and exports that for ci-gate. Its @@ -1783,6 +1796,46 @@ jobs: build-artifact-name: bazel-ci-build-artifacts integration: "on" + # D2 step 1: pr-risk.yml's bazel-embedded-coverage job, verbatim (policy- + # tested identical). Where it says covered, PR Risk skips its legacy + # embedded test jobs, so ci-gate requires the Bazel embedded lane to have + # run remotely and passed in this run. + bazel-embedded-coverage: + name: Bazel embedded coverage + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + covered: ${{ steps.decide.outputs.covered }} + steps: + # GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's + # expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS + # and not the executor secret (either, evaluated per run, would let one + # workflow's re-run disagree with the other's). A same-repo PR whose run + # lacks the secret is covered anyway, so pr.yml's gate goes red (mode + # local) instead of the tier silently moving back. Dependabot runs get + # no Actions secrets by design and keep the legacy tier; github.actor + # (unlike triggering_actor) stays the PR's actor on a re-run. + - name: Decide whether the Bazel lane covers the embedded tier + id: decide + env: + RETIRED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} + PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + FORK: ${{ github.event.pull_request.head.repo.fork == true }} + DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} + run: | + for v in RETIRED PULL_REQUEST FORK DEPENDABOT; do + case "${!v}" in + true|false) ;; + *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; + esac + done + covered=false + if [ "$RETIRED" = true ] && [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then + covered=true + fi + echo "covered=$covered" >> "$GITHUB_OUTPUT" + echo "Legacy embedded Dolt tier retired in favour of the Bazel lane: **$covered** (retired: $RETIRED, pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT)" | tee -a "$GITHUB_STEP_SUMMARY" + ci-gate: name: CI Gate / Required runs-on: ubuntu-latest @@ -1811,6 +1864,7 @@ jobs: - lint - windows-make-shell - bazel + - bazel-embedded-coverage if: ${{ always() }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1850,6 +1904,8 @@ jobs: BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE + BAZEL_EMBEDDED_COVERAGE + BAZEL_EMBEDDED_RETIRED BUILD_ARTIFACTS: ${{ needs.build-artifacts.result }} CHECK_BUILD_TAGS: ${{ needs.check-build-tags.result }} CHECK_CMD_BD_PUREGEO_TESTS: ${{ needs.check-cmd-bd-puregeo-tests.result }} @@ -1889,6 +1945,10 @@ jobs: BAZEL_CALL: ${{ needs.bazel.result }} BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} BAZEL_RBE_ENABLED: ${{ needs.bazel.outputs.rbe-enabled }} + # D2 step 1: whether PR Risk retired its legacy embedded tier for + # this PR (BAZEL_EMBEDDED_RETIRED below). + BAZEL_EMBEDDED_COVERAGE: ${{ needs.bazel-embedded-coverage.result }} + BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-embedded-coverage.outputs.covered }} run: | skipped_ok="" if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then @@ -1901,5 +1961,15 @@ jobs: skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)" BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)" export BAZEL + # Where PR Risk's legacy embedded tier is retired, this run's Bazel + # embedded lane is the tier's only pre-merge run: it must have run + # remotely and passed. Mode skip (the farm switch off), or any + # other result, is red whatever bazel-gate.sh allows. + BAZEL_EMBEDDED_RETIRED=success + if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]] && { [[ "$BAZEL_RBE_MODE" != "remote" ]] || [[ "$BAZEL_EMBEDDED" != "success" ]]; }; then + BAZEL_EMBEDDED_RETIRED="not-run-remotely" + echo "::error::PR Risk's legacy embedded Dolt tier is retired (BAZEL_RETIRES_LEGACY_EMBEDDED) and the Bazel embedded lane did not run remotely and pass (mode '${BAZEL_RBE_MODE:-none}', result '${BAZEL_EMBEDDED:-none}'). If remote execution is off on purpose, commit BAZEL_RETIRES_LEGACY_EMBEDDED: \"false\" in pr.yml and pr-risk.yml and push this PR again." + fi + export BAZEL_EMBEDDED_RETIRED export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index c596e485a..94905d887 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -49,7 +49,9 @@ Current PR-related workflow names: call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. The legacy jobs these mirror stay required in - `pr.yml` and `pr-risk.yml`. + `pr.yml` and `pr-risk.yml`, except where `BAZEL_EMBEDDED` runs in their + place (see + [Legacy Embedded Tier Retirement](#legacy-embedded-tier-retirement-d2-step-1)). `.github/scripts/bazel-gate.sh` reads the exported mode, never the variable or the fork flag: mode `skip` allows every Bazel id to skip, mode `local` allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, @@ -77,10 +79,16 @@ Current PR-related workflow names: red, and autofix only patches packages the PR itself changed), the RBE farm is down, or the beads CI RBE client certificate expires (about 2027-09-27; a partial secret set also fails every same-repo run), fix - `main` (`make bazel-sync`) or renew the secrets, or unset the - `RBE_WEST_WORKERS` repo variable: same-repo runs then take mode `skip`, - which the gate accepts (fork PRs still run locally, so drift on `main` - still reaches them). + `main` (`make bazel-sync`) or renew the secrets. To turn remote execution + off instead, first commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"` in both + `pr.yml` and `pr-risk.yml` (see + [Legacy Embedded Tier Retirement](#legacy-embedded-tier-retirement-d2-step-1)), + then unset the `RBE_WEST_WORKERS` repo variable: same-repo runs then take + mode `skip`, which the gate accepts (fork PRs still run locally, so drift + on `main` still reaches them). Unsetting the variable while the flag is + still `"true"` turns `CI Gate / Required` red on every same-repo PR + (`BAZEL_EMBEDDED_RETIRED`), because PR Risk no longer runs the legacy + embedded tier for them. - `.github/workflows/bazel-farm.yml`: `Bazel Farm (trusted forks)` Runs on `pull_request_target` for fork PRs to `main` whose author and triggering user are on `.github/bazel-farm-allowlist.txt`, and calls @@ -89,8 +97,9 @@ Current PR-related workflow names: [Trusted-Author Fork PRs](#trusted-author-fork-prs-bazel-farm). - `.github/workflows/pr-risk.yml`: `PR Risk` Runs on `pull_request` and `merge_group`. Contains embedded Dolt risk - detection, embedded build/test shards, the Nix flake smoke check, and the - risk aggregate gate `PR Risk / PR Risk Gate / Required`. + detection, the `bazel-embedded-coverage` decision, embedded build/test + shards, the proxied and server Dolt shards, the Nix flake smoke check, and + the risk aggregate gate `PR Risk / PR Risk Gate / Required`. - `.github/workflows/main.yml`: `Main` Runs on pushes to `main`. Contains the main branch health checks, package gates, platform smoke/short coverage, embedded Dolt coverage, and promoted @@ -121,8 +130,12 @@ the `bazel` call) and `PR Risk Gate / Required` (`pr-risk.yml`). PR Risk's gate job was renamed from `CI Gate / Required` to `PR Risk Gate / Required` (#6939), so the two required contexts are distinct; before that both workflows reported the same check name. Organization admins and one team -may bypass it. There is no merge queue, so the checks run on -`pull_request`. +may bypass it. There is no merge queue (no `merge_queue` rule, and no +`merge_group` run has ever happened), so the checks run on `pull_request` +only, and the ruleset does not require branches to be up to date +(strict off). The ruleset applies to the default branch only: PRs into +`release/**` run both workflows (and the Bazel lane), but neither gate is +enforced there. ## Trusted-Author Fork PRs (Bazel Farm) @@ -420,7 +433,8 @@ would only be needed if maintainers still want exactly one required check. - Baseline aggregate candidate: `PR / CI Gate / Required` - Risk aggregate candidate: `PR Risk / PR Risk Gate / Required` - Source: GitHub Actions -- Required on: pull requests and merge queue groups targeting `main` +- Required on: pull requests (and merge queue groups, if a queue is ever + added) targeting `main`; not on `release/**` Do not require these existing check names directly: @@ -433,8 +447,10 @@ Do not require these existing check names directly: - `Test (ubuntu-latest)` - `Test (macos-latest)` - `Test (storage domain + uow)` +- `Bazel embedded coverage` - `Build (Embedded Dolt)` - `Test (Embedded Dolt Storage 1/5)` through `Test (Embedded Dolt Storage 5/5)` +- `Test (Embedded Dolt Conformance - core)` and `- audit` - `Test (Embedded Dolt Cmd 1/20)` through `Test (Embedded Dolt Cmd 20/20)` - `Test (Windows - smoke)` - `Check formatting` @@ -554,7 +570,8 @@ intentionally not kept in lockstep with later leaf additions and renames; use `.github/workflows/pr-risk.yml` has a companion aggregate for `detect-ci-tier`, -`build-embedded`, `test-embedded-storage`, `test-embedded-cmd`, and `test-nix`. +`bazel-embedded-coverage`, `build-embedded`, the embedded, proxied and server +Dolt test jobs, and `test-nix`. `.github/scripts/ci-gate.sh` is a small shell evaluator. It fails on any `failure` or `cancelled` result. It accepts `skipped` only for jobs that are @@ -562,8 +579,15 @@ intentionally absent for that event or risk tier: - `CHECK_NO_BEADS_CHANGES=skipped` is acceptable on `merge_group` because the job is PR-only. -- In the risk aggregate, `BUILD_EMBEDDED`, `TEST_EMBEDDED_STORAGE`, and - `TEST_EMBEDDED_CMD` may be `skipped` only when `FULL_EMBEDDED != true`. +- In the risk aggregate, `BUILD_EMBEDDED` and the embedded, proxied and server + Dolt test ids may be `skipped` when `FULL_EMBEDDED != true`. +- In the risk aggregate, `TEST_EMBEDDED_STORAGE`, `TEST_EMBEDDED_CONFORMANCE` + and `TEST_EMBEDDED_CMD` may also be `skipped` when + `bazel-embedded-coverage` reported `covered=true`. `BAZEL_EMBEDDED_COVERAGE` + (that job's result) must be `success`. +- In the baseline aggregate, `BAZEL_EMBEDDED_COVERAGE` must be `success` and + `BAZEL_EMBEDDED_RETIRED` is red when `covered=true` but the Bazel embedded + lane did not run remotely and pass. - All baseline jobs must be `success`. This keeps branch protection pointed at stable aggregate jobs while preserving @@ -614,13 +638,144 @@ listed paths can be blocked waiting for a check that GitHub never creates. The current embedded Dolt topology already fits the required-check model: - `detect-ci-tier` always runs. -- `build-embedded`, `test-embedded-storage`, and `test-embedded-cmd` use - job-level `if`. +- `build-embedded`, `test-embedded-storage`, `test-embedded-conformance` and + `test-embedded-cmd` use job-level `if`; the three test jobs also stand down + where the Bazel lane covers them (next section). - `.github/scripts/ci-embedded-tier.sh` runs full embedded coverage for `push`, `merge_group`, unavailable PR diff bounds, and risky paths. - Docs-only PRs can skip the embedded matrix without leaving the required gate pending, because the aggregate job still runs. +### Legacy Embedded Tier Retirement (D2 Step 1) + +Since D2 step 1, `pr-risk.yml`'s legacy embedded test jobs +(`test-embedded-storage` x5, `test-embedded-conformance` x2, +`test-embedded-cmd` x20: 27 jobs) do not run on same-repo PRs. There, +`pr.yml`'s gated Bazel lane `Bazel / embedded Dolt tier` (`bazel.yml`'s +`bazel-embedded`, `--config=embedded`, the same tests and shard manifests) +is the tier's only pre-merge run, and `CI Gate / Required` requires it to +have run remotely and passed. + +- Switch: the committed workflow env `BAZEL_RETIRES_LEGACY_EMBEDDED`, set to + the same literal (`"true"` or `"false"`) in `pr.yml` and `pr-risk.yml` + (policy-tested). It is deliberately not the `RBE_WEST_WORKERS` repo + variable. A variable is read again by every run and re-run, and the two + workflows are separate runs, so one could see it on and the other off; + see "Why a committed flag" below. +- Who: `pull_request` runs from same-repo branches (not forks) whose + `github.actor` is not `dependabot[bot]`, while the flag is `"true"`. The + executor secret is deliberately not part of the decision: a same-repo PR + whose run lacks it (secret deleted or emptied) is still covered, takes + Bazel mode `local` (no embedded lane) and so turns `CI Gate / Required` + red, instead of quietly moving back to a legacy tier that one of its + runs may already have skipped. +- Everyone else keeps the legacy tier unchanged: + - fork PRs; + - Dependabot PRs (no Actions secrets; `github.actor`, unlike + `github.triggering_actor`, stays `dependabot[bot]` when someone else + re-runs them); + - every `merge_group` run (there is no merge queue today, so this is not + a pre-merge net); + - every PR while the flag is `"false"`. + + `main.yml`'s embedded jobs on push to `main` are untouched. +- How: + - `pr-risk.yml` and `pr.yml` each run the identical + `bazel-embedded-coverage` job (policy-tested). The job does no checkout + and runs no repository code. It reads the flag, the event, the fork flag + and the actor (no secret, no variable), and outputs `covered`. + - The three legacy test jobs add + `needs.bazel-embedded-coverage.outputs.covered != 'true'` to their `if`. + - PR Risk's gate accepts their skip only when `covered == true`. + - `pr.yml`'s gate requires `BAZEL_EMBEDDED_COVERAGE` (the job's result) + and `BAZEL_EMBEDDED_RETIRED`. The latter is red when `covered == true` + and the Bazel call's mode is not `remote` or `BAZEL_EMBEDDED` is not + `success`. In particular, mode `skip` (the farm switch off) and mode + `local` (the executor secret missing) are red there, although + `bazel-gate.sh` alone would accept them. + - A failed, cancelled or missing decision is red in both gates. +- `build-embedded` keeps running: its `embedded-test-binaries` artifact also + feeds `test-proxied-cmd`, `test-server-storage` and + `test-server-storage-full`, which are not retired in this step. +- Why a committed flag: + - `covered` depends on nothing a re-run can change (flag, event, fork, + actor). So any run or re-run of PR Risk that skips the legacy tier is + matched by `pr.yml` runs that compute the same `covered`. + - Each such `pr.yml` run is green only if the Bazel lane ran remotely and + passed in that run. + - Flipping `RBE_WEST_WORKERS` either way, plus "Re-run all jobs" of + either workflow, can no longer leave both required checks green with + neither tier run. + - `scripts/pr_risk_bazel_coverage_test.go` checks this by running both + workflows' actual decision steps, `bazel.yml`'s `rbe` step and `pr.yml`'s + actual gate step over every event, variable value, secret, fork, actor + and flag combination, with the variable and the secret each differing + between the two runs. It also requires the two workflows to share their + `pull_request` triggers. +- Cost of failing closed: while the flag is `"true"`, a same-repo PR without + the executor secret (secret deleted or emptied, or a non-Dependabot bot + whose runs get no secrets) is red until the secret is restored or the + flag is committed `"false"`. +- Revert: + 1. Commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"` in both workflows on + `main`. + 2. Every PR needs a new push or a merge of `main` to pick it up: a + pull_request run uses the workflow files of the PR's merge commit, and + a re-run reuses that commit. + 3. Only then unset `RBE_WEST_WORKERS`, if remote execution should be off. + + Reverting the D2 commits removes the decision jobs entirely. +- Admin changes: flipping `RBE_WEST_WORKERS` or deleting the executor + secret, followed by "Re-run all jobs" of either workflow, can no longer + leave both required checks green with neither tier run. +- Re-runs: "Re-run all jobs" (or "Re-run failed jobs") of one workflow + re-evaluates variables and secrets for that workflow's run only. The + other workflow's last result stays on the head SHA. With the committed + flag this cannot drop the tier silently: `covered` depends on the flag + (fixed by the merge commit), the event, the fork flag and the actor, and + on no variable or secret. Re-running "failed jobs" keeps the earlier + jobs' outputs, including `covered` and the `rbe` mode. +- Enforcement scope: the beads-only ruleset requires both gates on the + default branch (`main`) only. On PRs into `release/**` both workflows run + and report, but merging does not wait for them, before or after this + change. There is no merge queue and `strict` is off, so no + pre-merge run catches a gap; only `main.yml`'s embedded jobs and + `bazel.yml`'s push run do, after merge. +- Lane hardening that the retirement relies on (policy-tested in + `scripts/ci_workflow_test.go` and `scripts/pr_risk_bazel_coverage_test.go`): + - `test:embedded --nocache_test_results`: every run executes every test, + like the legacy `-test.count=1` jobs, so a stale or poisoned entry in + the shared farm action cache cannot stand in for a run. + - No retries: no `--flaky_test_attempts` or + `--runs_per_test_detects_flakes` anywhere, and no `flaky =` other than + a literal `False`. The lane also runs + `bazel query 'attr(flaky, 1, tests(//...))'` before the tier and fails + on any result, so a macro or variable cannot hide a flaky target. + - Pinned selection: + - The lane's `bazel test //... --config=embedded` step and the + `--config=embedded` lines are pinned exactly. + - No other rc line of a config the lane uses (unconfigured, + `remote-exec`, or anything they reference) may filter, narrow, + re-run or re-route tests. + - `.bazelrc` imports only the two gitignored try-imports, and no other + rc file is committed. + - `tools/bazel/*.sh` and the whole setup-bazel action may not inject + test selection or skips. + - The four embedded-tagged targets' `args` and `env` are pinned. + - `tools/bazel/check_shard_coverage.py` runs after the tier. It requires: + - every Bazel shard to have run exactly the tests its shard script lists + (list-only mode, minus `TestMain`, which `grep '^func Test'` lists + but which is never a test); + - no shard, and no conformance partition, to be all skipped. + + A test the scripts discover from source but the Bazel binary lacks + fails the lane instead of passing silently, and so does an injected + `-test.short` or `BEADS_TEST_SKIP` that skips a whole shard. +- Not changed: `conformance.yml`'s Tier 1 (`scripts/conformance.sh`) runs the + embedded-Dolt `TestConformance` again (non-race, unsharded), duplicating + `test-embedded-conformance` and the Bazel lane. It is not part of either + required gate; retiring it is a separate decision. + ### Server Dolt Storage Matrix `test-server-storage-full` mirrors `test-embedded-storage`'s sharding, one @@ -691,6 +846,10 @@ Policy for `merge_group`: - `PR` and `PR Risk` must include `merge_group`. - `detect-ci-tier` should keep treating `merge_group` as full embedded coverage. +- There is no merge queue today (see Current State), so these runs are not a + pre-merge safety net for anything PRs skip; in particular they are not one + for the retired legacy embedded tier (D2 step 1). `merge_group` keeps that + tier only so a queue added later starts with full coverage. - Any risk detector added to the required topology should default to run on `merge_group`, because the merge group commit may combine individually safe PRs into a risky integration state. diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 8e01933da..1d622a1d8 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -32,6 +32,7 @@ go_test( "host_tools_test.go", "install_dolt_test.go", "pr_preflight_test.go", + "pr_risk_bazel_coverage_test.go", "pull_dolt_image_test.go", "release_script_test.go", "test_script_test.go", diff --git a/scripts/ci_capability_selector_test.go b/scripts/ci_capability_selector_test.go index d794ecf9a..707487bf1 100644 --- a/scripts/ci_capability_selector_test.go +++ b/scripts/ci_capability_selector_test.go @@ -96,7 +96,7 @@ func TestCICapabilitySelectorWorkflowPreservesExistingAuthority(t *testing.T) { if err := yaml.Unmarshal(data, &workflow); err != nil { t.Fatalf("parse workflow: %v", err) } - wantJobs := map[string]bool{"detect-ci-tier": true, "build-embedded": true, "test-embedded-storage": true, "test-embedded-conformance": true, "test-server-storage": true, "test-server-storage-full": true, "test-embedded-cmd": true, "test-proxied-cmd": true, "test-nix": true, "ci-gate": true} + wantJobs := map[string]bool{"detect-ci-tier": true, "bazel-embedded-coverage": true, "build-embedded": true, "test-embedded-storage": true, "test-embedded-conformance": true, "test-server-storage": true, "test-server-storage-full": true, "test-embedded-cmd": true, "test-proxied-cmd": true, "test-nix": true, "ci-gate": true} if len(workflow.Jobs) != len(wantJobs) { t.Errorf("job count = %d, want %d", len(workflow.Jobs), len(wantJobs)) } @@ -118,18 +118,25 @@ func TestCICapabilitySelectorWorkflowPreservesExistingAuthority(t *testing.T) { if tier.ID != "tier" || shadow.ID != "shadow-selector" || !shadow.ContinueOnError || detector.stepIndex(t, shadow.Name) != detector.stepIndex(t, tier.Name)+1 { t.Errorf("shadow step is not immediately after tier: tier=%+v shadow=%+v", tier, shadow) } - for _, name := range []string{"build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd"} { + for _, name := range []string{"build-embedded", "test-server-storage", "test-server-storage-full", "test-proxied-cmd"} { if got := workflow.Jobs[name].If; got != "needs.detect-ci-tier.outputs.full_embedded == 'true'" { t.Errorf("%s if = %q", name, got) } } + // The legacy embedded test jobs also stand down where the gated Bazel + // lane covers them (TestPRRiskLegacyEmbeddedTierDefersToBazelLane). + for _, name := range prRiskBazelCoveredJobs { + if got := workflow.Jobs[name].If; got != prRiskBazelCoveredIf { + t.Errorf("%s if = %q, want %q", name, got, prRiskBazelCoveredIf) + } + } if workflow.Jobs["test-nix"].If != "" { t.Errorf("test-nix if = %q, want ungated", workflow.Jobs["test-nix"].If) } gate := workflow.Jobs["ci-gate"] - wantNeeds := []string{"detect-ci-tier", "build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd", "test-nix"} + wantNeeds := []string{"detect-ci-tier", "bazel-embedded-coverage", "build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd", "test-nix"} gateStep := gate.step(t, "Evaluate CI gate") - if gate.If != "${{ always() }}" || strings.Join(gate.Needs, ",") != strings.Join(wantNeeds, ",") || gateStep.Env["FULL_EMBEDDED"] != "${{ needs.detect-ci-tier.outputs.full_embedded }}" || gateStep.Env["CI_GATE_REQUIRED"] != "DETECT_CI_TIER BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD TEST_NIX" || !strings.Contains(gateStep.Run, "skipped_ok=\"BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD\"") { + if gate.If != "${{ always() }}" || strings.Join(gate.Needs, ",") != strings.Join(wantNeeds, ",") || gateStep.Env["FULL_EMBEDDED"] != "${{ needs.detect-ci-tier.outputs.full_embedded }}" || gateStep.Env["CI_GATE_REQUIRED"] != "DETECT_CI_TIER BAZEL_EMBEDDED_COVERAGE BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD TEST_NIX" || !strings.Contains(gateStep.Run, "skipped_ok=\"BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD\"") { t.Errorf("ci-gate authority changed: needs=%v env=%v", gate.Needs, gateStep.Env) } for name, job := range workflow.Jobs { diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index b3dbe74d2..853c09eac 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2048,7 +2048,8 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { if !contains(gate.Needs, "bazel") { t.Errorf("ci-gate needs = %v, want bazel", gate.Needs) } - wantBazelIDs := []string{bazelAggregateGateID} + // Plus D2 step 1's retirement check (TestPRRiskEmbeddedDecisionMatchesBazelMode). + wantBazelIDs := []string{bazelAggregateGateID, "BAZEL_EMBEDDED_COVERAGE", "BAZEL_EMBEDDED_RETIRED"} for lane, id := range bazelLaneGateIDs { wantBazelIDs = append(wantBazelIDs, id) if want := "${{ needs.bazel.outputs." + lane + " || 'skipped' }}"; evaluate.Env[id] != want { @@ -2101,7 +2102,9 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { } } - // Legacy jobs stay required: D1 adds, D2 removes. + // Legacy jobs stay required: D1 adds, D2 removes. (D2 step 1 lets + // pr-risk.yml's embedded test jobs skip where this lane runs remotely, + // but they stay required ids: TestPRRiskLegacyEmbeddedTierDefersToBazelLane.) for id, job := range map[string]string{ "BUILD_ARTIFACTS": "build-artifacts", "PR_CORE_WRAPPER": "pr-core-wrapper", @@ -2215,8 +2218,11 @@ type bazelGateScenario struct { mode, enabled string // the call's rbe-mode / rbe-enabled outputs call string // needs.bazel.result outputs map[string]string // the lanes' outputs ("" = not reported) - wantPass bool - wantMention string // a red gate must name this id + // pr.yml's bazel-embedded-coverage job (D2 step 1): its covered output + // and its result ("" = success). + covered, coverage string + wantPass bool + wantMention string // a red gate must name this id } // runPRGateStep runs pr.yml's actual "Evaluate CI gate" step (its run block, @@ -2239,6 +2245,13 @@ func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (boo } var got string switch { + case m[1] == prRiskCoverageJobName && m[2] == "result": + got = sc.coverage + if got == "" { + got = "success" + } + case m[1] == prRiskCoverageJobName && m[3] == "covered": + got = sc.covered case m[1] != "bazel" && m[2] == "result": got = "success" case m[1] != "bazel": @@ -3334,6 +3347,33 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { // runs the embedded variants race like the jobs' binaries, with the jobs' // parallelism; each shard variant runs its job's shard script with the job's // shard total, and the conformance variants pass the jobs' exact selectors. +// bazel-embedded's test step, exactly (review F4). +const bazelEmbeddedTestRun = `set -o pipefail +start=$(date +%s) +rc=0 +bazel test //... --config=embedded \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? +echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" +exit "$rc"` + +// .bazelrc's --config=embedded, exactly and in order (review F4): no +// --test_filter, no -test.short/-test.run/-test.skip, no retries, no +// result caching. The conformance targets' own -test.run/-test.skip args +// (the legacy jobs' partition) are checked against pr-risk.yml below. +var bazelEmbeddedRCLines = []string{ + "test:embedded --@rules_go//go/config:race", + "test:embedded --test_tag_filters=embedded", + "test:embedded --build_tests_only", + "test:embedded --keep_going", + "test:embedded --test_summary=terse", + "test:embedded --test_timeout=-1,-1,-1,1200", + "test:embedded --test_arg=-test.parallel=4", + "test:embedded --test_env=GO_TEST_WRAP_TESTV=1", + "test:embedded --remote_download_regex=.*/test\\.xml$", + "test:embedded --nocache_test_results", +} + func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) test := job.step(t, "bazel test //... --config=embedded") @@ -3344,6 +3384,13 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { t.Errorf("embedded step selects remote-exec itself; setup-bazel's rc does that only when secrets are present") } assertTestStepKeepsExitStatus(t, test) + // Review F4: the exact command line. Since D2 step 1 this lane is the + // tier's only pre-merge run on same-repo PRs, so an extra flag (a + // --test_filter, a --test_arg=-test.short/-test.run/-test.skip) that + // quietly narrows it must be a reviewed edit of this test too. + if strings.TrimSpace(test.Run) != bazelEmbeddedTestRun { + t.Errorf("embedded step run changed; want exactly:\n%s\ngot:\n%s", bazelEmbeddedTestRun, test.Run) + } // A shard with no tests assigned, or a selector that matches nothing, // exits 0; the job fails on any target or shard whose test.xml lists none. if !strings.Contains(test.Run, `--build_event_json_file="$RUNNER_TEMP/bazel-bep.json"`) { @@ -3365,12 +3412,43 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { "test:embedded --test_tag_filters=embedded", // The jobs pass no -parallel: GOMAXPROCS on 4-vCPU ubuntu-latest. "test:embedded --test_arg=-test.parallel=4", + // The tier's only pre-merge run (D2 step 1) must execute, like the + // legacy jobs' -test.count=1, never replay a cached result. + "test:embedded --nocache_test_results", } { if !rc[want] { t.Errorf(".bazelrc lacks %q", want) } } + // Review F4: --config=embedded is exactly these lines, in this order, and + // no unconfigured common/build/test line (which applies to every config) + // selects or narrows tests. + var embeddedLines []string + narrow := regexp.MustCompile(`test_filter|test_arg|-test\.(short|run|skip)|test_tag_filters|test_lang_filters|test_size_filters|test_timeout_filters`) + for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { + line = strings.TrimSpace(line) + cmd, _, _ := strings.Cut(line, " ") + if strings.HasSuffix(cmd, ":embedded") { + embeddedLines = append(embeddedLines, line) + } + if (cmd == "common" || cmd == "build" || cmd == "test") && narrow.MatchString(line) { + t.Errorf(".bazelrc %q narrows every config's tests, including --config=embedded", line) + } + } + if !reflect.DeepEqual(embeddedLines, bazelEmbeddedRCLines) { + t.Errorf(".bazelrc --config=embedded lines changed; want exactly:\n%s\ngot:\n%s", + strings.Join(bazelEmbeddedRCLines, "\n"), strings.Join(embeddedLines, "\n")) + } + if gen := readPolicyFile(t, sourceRepoRoot(t), setupBazelActionDir+"/write-bazelrc.sh"); narrow.MatchString(gen) { + t.Errorf("setup-bazel's generated rc selects or narrows tests; it may only configure remote execution") + } for line := range rc { + // Nothing turns result caching back on for the embedded lane (a + // later --cache_test_results wins over --nocache_test_results). + if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && + line != "test:embedded --nocache_test_results" && line != "test:docker --nocache_test_results" { + t.Errorf(".bazelrc %q: only test:embedded and test:docker set test result caching", line) + } if strings.HasPrefix(line, "test:embedded ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP")) { t.Errorf(".bazelrc %q: the embedded jobs run without -short and BEADS_TEST_SKIP", line) } @@ -4461,3 +4539,71 @@ func TestReleaseWorkflowRestoresNoCache(t *testing.T) { t.Fatal("release.yml has no setup-go step; update this test") } } + +// Review F3: the gated Bazel lanes never retry a failing test. The legacy +// jobs they mirror (and, since D2 step 1, replace for the embedded tier) ran +// each test once, so a retry would hide a flaky failure no other job sees. +// Nothing may set --flaky_test_attempts (or --runs_per_test_detects_flakes, +// which reports a failed-then-passed test as FLAKY, not FAILED): not .bazelrc, +// a workflow, setup-bazel's generated rc, or a tools/bazel wrapper; and no +// BUILD file or macro may mark a target flaky = True (Bazel retries those up +// to three times by default). +func TestBazelGatedLanesNeverRetryFlakyTests(t *testing.T) { + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("scripts_test's runfiles hold no other package's BUILD files") + } + root := sourceRepoRoot(t) + retry := regexp.MustCompile(`flaky_test_attempts|runs_per_test_detects_flakes`) + // Any flaky = other than a literal False/0 (a variable or macro + // parameter could be True). bazel-embedded also asks Bazel itself + // (TestBazelEmbeddedQueriesFlakyTargets). + flakyAttr := regexp.MustCompile(`\bflaky\s*=\s*([^,)\s]+)`) + checked := 0 + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(root, path) + if d.IsDir() { + switch d.Name() { + case ".git", "node_modules", ".beads": + return filepath.SkipDir + } + return nil + } + if d.Type()&os.ModeSymlink != 0 { + return nil // bazel-* convenience symlinks + } + base := d.Name() + isBuild := base == "BUILD" || base == "BUILD.bazel" || strings.HasSuffix(base, ".bzl") + isRetrySurface := rel == ".bazelrc" || strings.HasPrefix(rel, ".github"+string(filepath.Separator)) || + strings.HasPrefix(rel, filepath.Join("tools", "bazel")+string(filepath.Separator)) + if !isBuild && !isRetrySurface { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + checked++ + for i, line := range strings.Split(string(data), "\n") { + code := line + if j := strings.Index(code, "#"); j >= 0 && (isBuild || rel == ".bazelrc") { + code = code[:j] + } + if retry.MatchString(code) { + t.Errorf("%s:%d retries failing tests (%q); the gated lanes run each test once", rel, i+1, strings.TrimSpace(line)) + } + if m := flakyAttr.FindStringSubmatch(code); isBuild && m != nil && m[1] != "False" && m[1] != "0" { + t.Errorf("%s:%d marks a target flaky (%q); Bazel would retry it", rel, i+1, strings.TrimSpace(line)) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if checked < 10 { + t.Fatalf("checked only %d files; is the repository root right?", checked) + } +} diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go new file mode 100644 index 000000000..cc8dade13 --- /dev/null +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -0,0 +1,1169 @@ +package scripts_test + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// D2 step 1: PR Risk's legacy embedded-Dolt test jobs stand down on the PRs +// where pr.yml's gated Bazel `embedded Dolt tier` lane (bazel.yml's +// bazel-embedded, remote mode only) is the tier's run, and nowhere else. +// Both workflows run the same bazel-embedded-coverage job, which reads a +// committed flag (BAZEL_RETIRES_LEGACY_EMBEDDED), never the mutable +// RBE_WEST_WORKERS variable. PR Risk's gate accepts the legacy skips only +// when that job says covered; pr.yml's gate then requires the Bazel lane to +// have run remotely and passed, so no re-run of either workflow, with the +// variable flipped either way, can leave both gates green and the tier +// unrun. + +const ( + prRiskWorkflowName = "pr-risk.yml" + prRiskCoverageJobName = "bazel-embedded-coverage" + prRiskCoverageCovered = "${{ steps.decide.outputs.covered }}" + prRiskPullRequestValue = "${{ github.event_name == 'pull_request' }}" + prRiskRetiredFlag = "BAZEL_RETIRES_LEGACY_EMBEDDED" + prRiskRetiredValue = "${{ env." + prRiskRetiredFlag + " == 'true' }}" + // The legacy jobs' if: the existing risk tier, and not covered. + prRiskBazelCoveredIf = "needs.detect-ci-tier.outputs.full_embedded == 'true' && needs." + prRiskCoverageJobName + ".outputs.covered != 'true'" +) + +// The retired legacy jobs and their gate ids. build-embedded is not one: its +// artifact also feeds the proxied and server Dolt jobs. +var prRiskBazelCoveredJobs = []string{"test-embedded-storage", "test-embedded-conformance", "test-embedded-cmd"} + +var prRiskBazelCoveredIDs = map[string]string{ + "test-embedded-storage": "TEST_EMBEDDED_STORAGE", + "test-embedded-conformance": "TEST_EMBEDDED_CONFORMANCE", + "test-embedded-cmd": "TEST_EMBEDDED_CMD", +} + +// The bazel.yml rbe step env keys the decision copies verbatim. +var prRiskSharedDecisionEnv = []string{"FORK"} + +// The decision's Dependabot test: Dependabot runs get no Actions secrets, so +// bazel.yml runs them locally and they keep the legacy tier. +const prRiskDependabotValue = "${{ github.actor == 'dependabot[bot]' }}" + +// rbeFacts: what GitHub evaluates the decision steps' env expressions on. +type rbeFacts struct { + event string // github.event_name + rbeVar string // vars.RBE_WEST_WORKERS ("" = unset) + secret string // secrets.RBE_WEST_EXECUTOR ("" = unavailable: fork, Dependabot, unset) + fork bool // github.event.pull_request.head.repo.fork + retired string // the committed env.BAZEL_RETIRES_LEGACY_EMBEDDED + // github.actor is dependabot[bot] (fixed for a PR's runs and re-runs). + dependabot bool +} + +func (f rbeFacts) String() string { + return fmt.Sprintf("event=%s var=%q secret=%v fork=%v retired=%q dependabot=%v", f.event, f.rbeVar, f.secret != "", f.fork, f.retired, f.dependabot) +} + +var ( + rbeEvents = []string{"pull_request", "merge_group", "push", "workflow_dispatch", "pull_request_target"} + rbeVarValues = []string{"", "true", "True", "TRUE", "false", "1", "yes"} + rbeSecrets = []string{"", "grpcs://rbe.example:443"} + retiredValues = []string{"true", "True", "false", ""} +) + +// rbeFactsMatrix: every combination of the facts the decisions read. +func rbeFactsMatrix() []rbeFacts { + var out []rbeFacts + for _, event := range rbeEvents { + for _, v := range rbeVarValues { + for _, secret := range rbeSecrets { + for _, fork := range []bool{false, true} { + for _, retired := range retiredValues { + for _, dependabot := range []bool{false, true} { + out = append(out, rbeFacts{event, v, secret, fork, retired, dependabot}) + } + } + } + } + } + } + return out +} + +// evalRBEExpr evaluates the env expressions the decision steps may use, for +// a bazel.yml call with these inputs (with: the caller's `with:`; unset +// inputs take their defaults). GitHub's == on strings is case-insensitive. +// Any other expression fails the test, so the simulation cannot silently +// drift from the workflows. +func evalRBEExpr(t *testing.T, expr string, f rbeFacts, with map[string]string) string { + t.Helper() + input := func(name, def string) string { + if v, ok := with[name]; ok { + return v + } + return def + } + switch expr { + case prRiskPullRequestValue: + return strconv.FormatBool(f.event == "pull_request") + case prRiskRetiredValue: + return strconv.FormatBool(strings.EqualFold(f.retired, "true")) + case prRiskDependabotValue: + return strconv.FormatBool(f.dependabot) + case "${{ vars.RBE_WEST_WORKERS == 'true' }}": + return strconv.FormatBool(strings.EqualFold(f.rbeVar, "true")) + case "${{ inputs.rbe == 'off' }}": + return strconv.FormatBool(strings.EqualFold(input("rbe", "on"), "off")) + case "${{ github.event.pull_request.head.repo.fork == true }}": + return strconv.FormatBool(f.fork) + case bazelForkFarmValue: + return strconv.FormatBool(strings.EqualFold(input("fork-farm", "off"), "authorized") && + f.event == "pull_request_target" && input("checkout-sha", "") != "") + case bazelRBESecretValue: + return strconv.FormatBool(f.secret != "") + } + t.Fatalf("decision env expression %q: teach evalRBEExpr how GitHub evaluates it", expr) + return "" +} + +// runDecisionStep runs a decision step's script under GitHub's bash flags +// with its env evaluated for f, and returns its $GITHUB_OUTPUT. +func runDecisionStep(t *testing.T, step ciWorkflowStep, f rbeFacts, with map[string]string) (map[string]string, error) { + t.Helper() + env := map[string]string{} + for k, v := range step.Env { + env[k] = evalRBEExpr(t, v, f, with) + } + return runBazelRBEDecision(t, step.Run, env) +} + +func coverageStep(t *testing.T, workflow string) ciWorkflowStep { + t.Helper() + job := readCIWorkflow(t, workflow).job(t, prRiskCoverageJobName) + if len(job.Steps) != 1 { + t.Fatalf("%s %s has %d steps, want exactly the decision step", workflow, prRiskCoverageJobName, len(job.Steps)) + } + return job.Steps[0] +} + +func prRiskCoverageStep(t *testing.T) ciWorkflowStep { return coverageStep(t, prRiskWorkflowName) } + +// workflowEnv: a workflow's top-level env. +func workflowEnv(t *testing.T, name string) map[string]string { + t.Helper() + var doc struct { + Env map[string]string `yaml:"env"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+name)), &doc); err != nil { + t.Fatal(err) + } + return doc.Env +} + +// The decision job reads the committed flag, the event, the actor and, +// through bazel.yml's rbe job's own expression, the fork flag: nothing a +// re-run can change, so never RBE_WEST_WORKERS, any other variable or any +// secret, and it runs no repository code. pr.yml runs the identical job, and +// both workflows commit the same flag. Nothing else in pr-risk.yml reads the +// facts, and nothing in it reads a secret. +func TestPRRiskBazelEmbeddedCoverageJob(t *testing.T) { + risk := readCIWorkflow(t, prRiskWorkflowName) + job := risk.job(t, prRiskCoverageJobName) + if len(job.Needs) != 0 || job.If != "" || job.RunsOn != "ubuntu-latest" || len(job.Env) != 0 || job.ContinueOnError || job.TimeoutMinutes == 0 { + t.Errorf("%s: needs %v, if %q, runs-on %q, env %v, continue-on-error %v, timeout %d; want no needs, if, env or continue-on-error, ubuntu-latest, a timeout", + prRiskCoverageJobName, job.Needs, job.If, job.RunsOn, job.Env, job.ContinueOnError, job.TimeoutMinutes) + } + if want := map[string]string{"covered": prRiskCoverageCovered}; !reflect.DeepEqual(job.Outputs, want) { + t.Errorf("%s outputs = %v, want %v", prRiskCoverageJobName, job.Outputs, want) + } + if prJob := readCIWorkflow(t, "pr.yml").job(t, prRiskCoverageJobName); !reflect.DeepEqual(prJob, job) { + t.Errorf("pr.yml's %s differs from pr-risk.yml's:\n%+v\n%+v", prRiskCoverageJobName, prJob, job) + } + riskFlag, prFlag := workflowEnv(t, prRiskWorkflowName)[prRiskRetiredFlag], workflowEnv(t, "pr.yml")[prRiskRetiredFlag] + if riskFlag != prFlag || (riskFlag != "true" && riskFlag != "false") { + t.Errorf("%s: pr-risk.yml %q, pr.yml %q; want the same literal \"true\" or \"false\" in both", prRiskRetiredFlag, riskFlag, prFlag) + } + step := prRiskCoverageStep(t) + rbeStep := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName).Steps[0] + wantEnv := map[string]string{"PULL_REQUEST": prRiskPullRequestValue, "RETIRED": prRiskRetiredValue, "DEPENDABOT": prRiskDependabotValue} + for _, k := range prRiskSharedDecisionEnv { + wantEnv[k] = rbeStep.Env[k] + } + if step.ID != "decide" || step.Uses != "" || step.Shell != "" || len(step.With) != 0 || step.If != "" || step.ContinueOnError != nil || !reflect.DeepEqual(step.Env, wantEnv) { + t.Errorf("%s step: id %q, uses %q, shell %q, with %v, if %q, env %v; want id decide, a plain run step with env %v", + prRiskCoverageJobName, step.ID, step.Uses, step.Shell, step.With, step.If, step.Env, wantEnv) + } + if strings.Contains(step.Run, "${{") || regexp.MustCompile(`\.github/|\./|source |\bbash\b|RBE_WEST_WORKERS|RBE_VAR|EXECUTOR`).MatchString(step.Run) { + t.Errorf("%s step runs repository code, interpolates expressions or reads the RBE variable or secret:\n%s", prRiskCoverageJobName, step.Run) + } + for k, v := range step.Env { + if regexp.MustCompile(`\b(secrets|vars)\s*(\.|\[)`).MatchString(v) { + t.Errorf("%s step env %s = %q reads a secret or variable; a re-run could change it", prRiskCoverageJobName, k, v) + } + } + + // Only the decision step reads the facts; nothing reads a secret or a + // repository variable. + stepEnv := ".jobs." + prRiskCoverageJobName + ".steps[0].env." + secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) + facts := regexp.MustCompile(`(?i)RBE_WEST_WORKERS|\bvars\s*(\.|\[)|head\.repo\.fork|RBE_WEST_EXECUTOR|github\.actor|dependabot|` + prRiskRetiredFlag) + walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", prRiskWorkflowName)), "", func(path string, key bool, value string) { + if key { + if value == prRiskRetiredFlag && path != ".env."+prRiskRetiredFlag { + t.Errorf("%s: %s sets %s; only the workflow env may", prRiskWorkflowName, path, prRiskRetiredFlag) + } + return + } + if secretRef.MatchString(value) { + t.Errorf("%s: %s reads secrets (%q); PR Risk needs none", prRiskWorkflowName, path, value) + } + if facts.MatchString(value) && !strings.HasPrefix(path, stepEnv) && !strings.HasPrefix(path, ".jobs."+prRiskCoverageJobName+".steps[0].run") { + t.Errorf("%s: %s re-derives the Bazel coverage decision (%q); read needs.%s.outputs.covered", prRiskWorkflowName, path, value, prRiskCoverageJobName) + } + }) +} + +// prGateFor: pr.yml's ci-gate scenario for one run whose Bazel call took +// this mode with every lane that runs in it passing, and whose +// bazel-embedded-coverage job said covered. +func prGateFor(t *testing.T, lanes map[string]map[string]bool, event, mode, covered string) bazelGateScenario { + t.Helper() + outputs := map[string]string{} + for lane, modes := range lanes { + if modes[mode] { + outputs[lane] = "success" + } + } + return bazelGateScenario{ + name: fmt.Sprintf("%s mode %s covered %s", event, mode, covered), event: event, + mode: mode, enabled: strconv.FormatBool(mode == "remote"), call: "success", + outputs: outputs, covered: covered, + } +} + +// bazelPRCallLanes: the modes each bazel.yml lane runs in under pr.yml's call. +func bazelPRCallLanes(t *testing.T, with map[string]string) map[string]map[string]bool { + t.Helper() + lanes := map[string]map[string]bool{} + for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if name != bazelRBEJobName { + lanes[name] = bazelLaneRunModes(t, name, job.If, with) + } + } + return lanes +} + +// Never both gates green with neither the legacy tier nor the Bazel lane +// having run it. covered (both workflows' actual decision scripts) is the +// committed flag on a same-repo, non-Dependabot pull_request: nothing a +// re-run can change. Across two runs (PR Risk's and pr.yml's, or a re-run of +// either) that see RBE_WEST_WORKERS and the executor secret differently, +// every combination: if PR Risk skipped the legacy tier, pr.yml's actual +// gate step is green only if the lane ran remotely. The happy path (flag, +// variable and secret on) is green with the legacy tier skipped; the kill +// switch (variable off) or a missing secret with the flag still on is red. +func TestPRRiskEmbeddedDecisionMatchesBazelMode(t *testing.T) { + requireHostTool(t, "bash") + pr := readCIWorkflow(t, "pr.yml") + call := pr.job(t, "bazel") + if call.Uses != "./.github/workflows/"+bazelWorkflowName { + t.Fatalf("pr.yml bazel job uses %q, want the local %s", call.Uses, bazelWorkflowName) + } + riskStep, prStep := prRiskCoverageStep(t), coverageStep(t, "pr.yml") + rbeStep := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName).Steps[0] + gateStep := pr.job(t, "ci-gate").step(t, "Evaluate CI gate") + lanes := bazelPRCallLanes(t, call.With) + + // The Bazel lane PR Risk defers to: remote-only, gated by pr.yml. + if !lanes[bazelEmbedJobName]["remote"] { + t.Fatalf("%s does not run in mode remote", bazelEmbedJobName) + } + gate := pr.job(t, "ci-gate") + required := strings.Fields(gateStep.Env["CI_GATE_REQUIRED"]) + for _, id := range []string{bazelLaneGateIDs[bazelEmbedJobName], "BAZEL_EMBEDDED_COVERAGE", "BAZEL_EMBEDDED_RETIRED"} { + if !contains(required, id) { + t.Errorf("pr.yml's ci-gate does not require %s", id) + } + } + if !contains(gate.Needs, "bazel") || !contains(gate.Needs, prRiskCoverageJobName) { + t.Errorf("pr.yml's ci-gate needs %v, want bazel and %s", gate.Needs, prRiskCoverageJobName) + } + cmd := exec.Command("bash", filepath.Join(sourceRepoRoot(t), bazelGateScript), "skips") + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=remote", "BAZEL_RBE_ENABLED=true"} + out, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + if contains(strings.Fields(string(out)), bazelLaneGateIDs[bazelEmbedJobName]) { + t.Fatalf("pr.yml's gate accepts a skipped %s in mode remote", bazelLaneGateIDs[bazelEmbedJobName]) + } + + // Both workflows run for the same PRs, so a PR Risk run that defers + // always has a pr.yml run that gates the lane. + type triggers struct { + On struct { + PullRequest struct { + Branches []string `yaml:"branches"` + Paths []string `yaml:"paths"` + Ignore []string `yaml:"paths-ignore"` + Types []string `yaml:"types"` + } `yaml:"pull_request"` + } `yaml:"on"` + } + var prOn, riskOn triggers + for name, dst := range map[string]*triggers{"pr.yml": &prOn, prRiskWorkflowName: &riskOn} { + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+name)), dst); err != nil { + t.Fatal(err) + } + } + if !reflect.DeepEqual(prOn, riskOn) || len(prOn.On.PullRequest.Paths)+len(prOn.On.PullRequest.Ignore) != 0 { + t.Fatalf("pull_request triggers differ or are path-filtered: pr.yml %+v, %s %+v", prOn, prRiskWorkflowName, riskOn) + } + + type decided struct{ covered, prCovered, mode string } + decideMemo := map[rbeFacts]decided{} + decide := func(t *testing.T, f rbeFacts) decided { + t.Helper() + if d, ok := decideMemo[f]; ok { + return d + } + bazel, err := runDecisionStep(t, rbeStep, f, call.With) + if err != nil { + t.Fatalf("bazel.yml rbe step: %v", err) + } + risk, err := runDecisionStep(t, riskStep, f, nil) + if err != nil || len(risk) != 1 { + t.Fatalf("%s decision step: %v %v", prRiskCoverageJobName, risk, err) + } + prd, err := runDecisionStep(t, prStep, f, nil) + if err != nil { + t.Fatalf("pr.yml %s step: %v", prRiskCoverageJobName, err) + } + d := decided{risk["covered"], prd["covered"], bazel["mode"]} + decideMemo[f] = d + return d + } + + // One run's facts: the decision itself. + sawCovered, sawLegacy := false, false + for _, f := range rbeFactsMatrix() { + d := decide(t, f) + want := strings.EqualFold(f.retired, "true") && f.event == "pull_request" && !f.fork && !f.dependabot + if d.covered != strconv.FormatBool(want) || d.prCovered != d.covered { + t.Errorf("%v: covered = %q (pr.yml %q), want %v", f, d.covered, d.prCovered, want) + } + if want { + sawCovered = true + } else { + sawLegacy = true + } + } + if !sawCovered || !sawLegacy { + t.Errorf("matrix never exercised both outcomes (covered %v, legacy %v)", sawCovered, sawLegacy) + } + + // Two runs (PR Risk's and pr.yml's, each possibly re-run) that agree on + // everything committed or fixed by the PR and differ in what an admin can + // change between them: the variable and the secret. + gateMemo := map[string]bool{} + prGatePasses := func(t *testing.T, event, mode, covered string) bool { + t.Helper() + key := event + "/" + mode + "/" + covered + if pass, ok := gateMemo[key]; ok { + return pass + } + pass, _ := runPRGateStep(t, gateStep, prGateFor(t, lanes, event, mode, covered)) + gateMemo[key] = pass + return pass + } + for _, f := range rbeFactsMatrix() { + if f.event != "pull_request" && f.event != "merge_group" { + continue // the events both workflows run on + } + risk := decide(t, f) + for _, v := range rbeVarValues { + for _, secret := range rbeSecrets { + g := f + g.rbeVar, g.secret = v, secret + prRun := decide(t, g) + legacyRan := risk.covered != "true" + bazelRan := prRun.mode == "remote" // and passed: every lane succeeds here + if !legacyRan && !bazelRan && prGatePasses(t, g.event, prRun.mode, prRun.prCovered) { + t.Errorf("PR Risk run %v skipped the legacy tier and pr.yml run (var %q, secret %v, mode %s, covered %s) is green without the Bazel lane", + f, v, secret != "", prRun.mode, prRun.prCovered) + } + } + } + } + + // Named cases, for the record (with the committed flag on). + for _, c := range []struct { + name string + f rbeFacts + covered string + prPasses bool + }{ + {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, "true", false}, "true", true}, + {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, "true", false}, "true", false}, + {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, "true", false}, "true", false}, + {"same-repo PR, flag reverted, var unset", rbeFacts{"pull_request", "", "x", false, "false", false}, "false", true}, + {"same-repo PR, flag reverted, secret missing", rbeFacts{"pull_request", "true", "", false, "false", false}, "false", true}, + {"fork PR", rbeFacts{"pull_request", "true", "", true, "true", false}, "false", true}, + {"fork PR somehow with a secret", rbeFacts{"pull_request", "true", "x", true, "true", false}, "false", true}, + {"Dependabot PR (no Actions secrets)", rbeFacts{"pull_request", "true", "", false, "true", true}, "false", true}, + {"Dependabot PR, var unset", rbeFacts{"pull_request", "", "", false, "true", true}, "false", true}, + {"merge_group", rbeFacts{"merge_group", "true", "x", false, "true", false}, "false", true}, + {"merge_group, var unset", rbeFacts{"merge_group", "", "x", false, "true", false}, "false", true}, + } { + d := decide(t, c.f) + if d.covered != c.covered { + t.Errorf("%s: covered = %q, want %s", c.name, d.covered, c.covered) + } + if pass, out := runPRGateStep(t, gateStep, prGateFor(t, lanes, c.f.event, d.mode, d.prCovered)); pass != c.prPasses { + t.Errorf("%s: pr.yml gate pass = %v, want %v\n%s", c.name, pass, c.prPasses, out) + } else if !pass && !regexp.MustCompile(`::error::BAZEL_EMBEDDED_RETIRED\b`).MatchString(out) { + t.Errorf("%s: red pr.yml gate does not name BAZEL_EMBEDDED_RETIRED:\n%s", c.name, out) + } + } + // Covered, remote, but the lane failed, was cancelled or reported + // nothing: red, naming the retirement too. + for _, res := range []string{"failure", "cancelled", ""} { + sc := prGateFor(t, lanes, "pull_request", "remote", "true") + sc.outputs[bazelEmbedJobName] = res + if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::BAZEL_EMBEDDED_RETIRED") { + t.Errorf("covered, embedded lane %q: gate pass = %v, want red naming BAZEL_EMBEDDED_RETIRED\n%s", res, pass, out) + } + } + // Covered, and an embedded result of success the mode cannot produce + // (the lane runs only in mode remote): the mode alone still makes it red. + for _, mode := range []string{"skip", "local"} { + sc := prGateFor(t, lanes, "pull_request", mode, "true") + sc.outputs[bazelEmbedJobName] = "success" + if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::BAZEL_EMBEDDED_RETIRED") { + t.Errorf("covered, mode %s, embedded reported success: gate pass = %v, want red naming BAZEL_EMBEDDED_RETIRED\n%s", mode, pass, out) + } + } + // pr.yml's coverage job failed: red even where nothing is retired. + for _, res := range []string{"failure", "cancelled", "skipped"} { + sc := prGateFor(t, lanes, "pull_request", "skip", "") + sc.coverage = res + if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::BAZEL_EMBEDDED_COVERAGE") { + t.Errorf("coverage job %s: gate pass = %v, want red naming BAZEL_EMBEDDED_COVERAGE\n%s", res, pass, out) + } + } + // A value that is not a boolean fails the job rather than deciding. + if out, err := runBazelRBEDecision(t, riskStep.Run, map[string]string{ + "RETIRED": "true", "PULL_REQUEST": "true", "FORK": "", "DEPENDABOT": "false", + }); err == nil { + t.Errorf("decision with FORK='' succeeded with %v; want failure", out) + } +} + +// prRiskGateScenario: what pr-risk.yml's ci-gate sees. +type prRiskGateScenario struct { + name string + results map[string]string // needs..result + outputs map[string]string // needs..outputs., keyed "job.name" + wantPass bool + wantMention string +} + +// runPRRiskGateStep runs pr-risk.yml's actual "Evaluate CI gate" step with its +// env evaluated for the scenario. An env expression of any other form fails +// the test. +func runPRRiskGateStep(t *testing.T, step ciWorkflowStep, sc prRiskGateScenario) (bool, string) { + t.Helper() + expr := regexp.MustCompile(`^\$\{\{ needs\.([A-Za-z0-9_-]+)\.(result|outputs\.([A-Za-z0-9_-]+)) \}\}$`) + env := []string{"PATH=" + os.Getenv("PATH"), "GITHUB_EVENT_NAME=pull_request"} + for key, value := range step.Env { + if !strings.Contains(value, "${{") { + env = append(env, key+"="+value) + continue + } + m := expr.FindStringSubmatch(value) + if m == nil { + t.Fatalf("pr-risk ci-gate env %s = %q: the gate simulation cannot evaluate it", key, value) + } + var got string + var ok bool + if m[2] == "result" { + got, ok = sc.results[m[1]] + } else { + got, ok = sc.outputs[m[1]+"."+m[3]], true + } + if !ok { + t.Fatalf("scenario %q has no result for needs.%s", sc.name, m[1]) + } + env = append(env, key+"="+got) + } + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step.Run) + cmd.Dir = sourceRepoRoot(t) + cmd.Env = env + out, err := cmd.CombinedOutput() + return err == nil, string(out) +} + +// The legacy embedded test jobs skip only when covered, and PR Risk's gate +// accepts their skip only then: the jobs' needs and if, the gate's wiring, +// and the actual gate step over every tier x decision, including a missing, +// failed or non-true decision with the jobs skipped (red). +func TestPRRiskLegacyEmbeddedTierDefersToBazelLane(t *testing.T) { + requireHostTool(t, "bash") + risk := readCIWorkflow(t, prRiskWorkflowName) + for _, name := range prRiskBazelCoveredJobs { + job := risk.job(t, name) + if job.If != prRiskBazelCoveredIf { + t.Errorf("%s if = %q, want %q", name, job.If, prRiskBazelCoveredIf) + } + if want := []string{"detect-ci-tier", prRiskCoverageJobName, "build-embedded"}; !reflect.DeepEqual([]string(job.Needs), want) { + t.Errorf("%s needs = %v, want %v", name, job.Needs, want) + } + } + // Every other job ignores the decision (build-embedded's artifact feeds + // the proxied and server Dolt jobs, which Bazel does not replace here). + covered := map[string]bool{} + for _, name := range prRiskBazelCoveredJobs { + covered[name] = true + } + for name, job := range risk.Jobs { + if covered[name] || name == "ci-gate" || name == prRiskCoverageJobName { + continue + } + if strings.Contains(job.If, prRiskCoverageJobName) || contains(job.Needs, prRiskCoverageJobName) { + t.Errorf("%s depends on %s; only %v may stand down", name, prRiskCoverageJobName, prRiskBazelCoveredJobs) + } + } + + gate := risk.job(t, "ci-gate") + step := gate.step(t, "Evaluate CI gate") + required := strings.Fields(step.Env["CI_GATE_REQUIRED"]) + if !contains(gate.Needs, prRiskCoverageJobName) || !contains(required, "BAZEL_EMBEDDED_COVERAGE") || + step.Env["BAZEL_EMBEDDED_COVERAGE"] != "${{ needs."+prRiskCoverageJobName+".result }}" || + step.Env["BAZEL_COVERS_EMBEDDED"] != "${{ needs."+prRiskCoverageJobName+".outputs.covered }}" { + t.Errorf("pr-risk ci-gate does not require %s's result and read its covered output: needs %v, env %v", prRiskCoverageJobName, gate.Needs, step.Env) + } + for _, id := range prRiskBazelCoveredIDs { + if !contains(required, id) { + t.Errorf("pr-risk ci-gate no longer requires %s (it runs wherever the Bazel lane does not)", id) + } + } + + // Each gated job's result for (tier, decision), as the jobs' if: produce + // it: needs.* outputs are strings, so only covered == 'true' skips. + results := func(full bool, coverageResult, coveredOut string) map[string]string { + r := map[string]string{} + for _, job := range gate.Needs { + r[job] = "success" + } + r[prRiskCoverageJobName] = coverageResult + for name := range risk.Jobs { + if !contains(gate.Needs, name) || name == "detect-ci-tier" || name == prRiskCoverageJobName || name == "test-nix" { + continue + } + runs := full + if covered[name] { + // A failed decision job skips its dependents. + runs = full && coverageResult == "success" && coveredOut != "true" + } + if !runs { + r[name] = "skipped" + } + } + return r + } + outputs := func(full bool, coveredOut string) map[string]string { + return map[string]string{ + "detect-ci-tier.full_embedded": strconv.FormatBool(full), + prRiskCoverageJobName + ".covered": coveredOut, + } + } + + var scenarios []prRiskGateScenario + for _, full := range []bool{true, false} { + for _, coveredOut := range []string{"true", "false"} { + name := fmt.Sprintf("full_embedded=%v covered=%s", full, coveredOut) + r := results(full, "success", coveredOut) + scenarios = append(scenarios, prRiskGateScenario{name: name + ", as designed", results: r, outputs: outputs(full, coveredOut), wantPass: true}) + for _, id := range prRiskBazelCoveredJobs { + if r[id] == "skipped" { + // Skipped by design; if it ran anyway and failed, red. + bad := copyMap(r) + bad[id] = "failure" + scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " ran and failed", bad, outputs(full, coveredOut), false, prRiskBazelCoveredIDs[id]}) + continue + } + for _, res := range []string{"skipped", "failure", "cancelled"} { + bad := copyMap(r) + bad[id] = res + scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " " + res, bad, outputs(full, coveredOut), false, prRiskBazelCoveredIDs[id]}) + } + } + if full { + // The decision never excuses the jobs it does not retire. + for _, id := range []string{"build-embedded", "test-proxied-cmd", "test-server-storage", "test-server-storage-full"} { + bad := copyMap(r) + bad[id] = "skipped" + scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " skipped", bad, outputs(full, coveredOut), false, ""}) + } + } + } + } + // A failed, cancelled or missing decision, or one that is not exactly + // 'true', excuses nothing, whatever the jobs did. + for _, bad := range []struct{ result, covered string }{ + {"failure", ""}, {"cancelled", ""}, {"skipped", ""}, {"success", ""}, {"success", "TRUE "}, {"success", "yes"}, {"success", "1"}, + } { + r := results(true, "success", "false") + r[prRiskCoverageJobName] = bad.result + for _, id := range prRiskBazelCoveredJobs { + r[id] = "skipped" + } + scenarios = append(scenarios, prRiskGateScenario{ + name: fmt.Sprintf("decision %s covered=%q, legacy skipped", bad.result, bad.covered), + results: r, outputs: outputs(true, bad.covered), wantPass: false, wantMention: "TEST_EMBEDDED_STORAGE", + }) + } + // The decision job itself must succeed even when nothing else needs it. + for _, res := range []string{"failure", "cancelled", "skipped"} { + r := results(false, "success", "false") + r[prRiskCoverageJobName] = res + scenarios = append(scenarios, prRiskGateScenario{"docs-only, decision " + res, r, outputs(false, ""), false, "BAZEL_EMBEDDED_COVERAGE"}) + } + + for _, sc := range scenarios { + t.Run(sc.name, func(t *testing.T) { + pass, out := runPRRiskGateStep(t, step, sc) + if pass != sc.wantPass { + t.Errorf("gate pass = %v, want %v (results %v, outputs %v)\n%s", pass, sc.wantPass, sc.results, sc.outputs, out) + } + if !sc.wantPass && sc.wantMention != "" && !regexp.MustCompile(`::error::`+sc.wantMention+`\b`).MatchString(out) { + t.Errorf("red gate does not name %s:\n%s", sc.wantMention, out) + } + }) + } +} + +func copyMap(m map[string]string) map[string]string { + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +// Review F5: bazel-embedded checks, after the run, that every Bazel shard of +// the manifest-sharded targets ran exactly the tests its PR Risk shard +// script lists, for the targets and shard counts of the legacy jobs. +func TestBazelEmbeddedChecksListedTestsRan(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) + step := job.step(t, "Every listed test ran in its shard") + risk := readCIWorkflow(t, prRiskWorkflowName) + want := []string{"python3 tools/bazel/check_shard_coverage.py", `--bep "$RUNNER_TEMP/bazel-bep.json"`} + for _, c := range []struct{ job, label, script string }{ + {"test-embedded-cmd", "//cmd/bd:bd_embedded_test", ".github/scripts/embedded-test-shard.sh"}, + {"test-embedded-storage", "//internal/storage/embeddeddolt:embeddeddolt_embedded_test", ".github/scripts/embedded-storage-test-shard.sh"}, + } { + shards := len(risk.job(t, c.job).Strategy.Matrix.Shard) + if !strings.Contains(risk.job(t, c.job).step(t, "Test").Run, c.script) { + t.Errorf("pr-risk.yml %s no longer runs %s; update this suite", c.job, c.script) + } + want = append(want, fmt.Sprintf("--suite %s %s %d", c.label, c.script, shards)) + } + // The conformance partitions (unsharded): not all skipped. + for _, partition := range []string{"core", "audit"} { + if risk.job(t, "test-embedded-conformance").step(t, "Test "+partition+" conformance").Run == "" { + t.Errorf("pr-risk.yml test-embedded-conformance has no %s partition; update this check", partition) + } + want = append(want, "--whole //internal/storage/embeddeddolt:embeddeddolt_conformance_"+partition+"_test") + } + if got := strings.Join(strings.Fields(step.Run), " "); got != strings.Join(want, " ") { + t.Errorf("%s coverage step runs %q, want %q", bazelEmbedJobName, got, strings.Join(want, " ")) + } + if step.If != "${{ always() && steps.test.outcome != 'skipped' }}" || step.ContinueOnError != nil { + t.Errorf("coverage step: if %q, continue-on-error %v; want always() after the test step and no continue-on-error", step.If, step.ContinueOnError) + } + if !(job.stepIndex(t, "bazel test //... --config=embedded") < job.stepIndex(t, step.Name) && + job.stepIndex(t, step.Name) < job.stepIndex(t, "Record job result")) { + t.Errorf("coverage step must run after the test step and before the result recorder") + } +} + +// check_shard_coverage.py itself, on a synthetic BEP, test.xml files and +// shard script. +func TestCheckShardCoverageScript(t *testing.T) { + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("scripts_test's runfiles hold no tools/bazel Python") + } + requireHostTool(t, "bash") + python := requireHostTool(t, "python3") + script := filepath.Join(sourceRepoRoot(t), "tools", "bazel", "check_shard_coverage.py") + dir := t.TempDir() + shard := filepath.Join(dir, "shard.sh") + // Shard 1 lists TestA and TestB, shard 2 TestC; list-only mode only. + if err := os.WriteFile(shard, []byte(`#!/usr/bin/env bash +set -euo pipefail +[ "${BEADS_TEST_SHARD_LIST_ONLY:-}" = 1 ] || { echo "not list-only" >&2; exit 3; } +[ "$2" = 2 ] || { echo "bad total $2" >&2; exit 1; } +echo "Shard $1/$2: running" +echo " manifest: 1, fallback: 0" +case "$1" in + 1) printf ' %s\n' TestA TestB TestMain ;; # TestMain: never a testcase + 2) printf ' %s\n' TestC ;; + *) exit 1 ;; +esac +[ -z "${FAKE_SHARD_FAIL:-}" ] || exit 1 +`), 0o755); err != nil { + t.Fatal(err) + } + const label = "//pkg:t" + writeBEP := func(shards int) string { + lines := []string{`{"id":{"targetConfigured":{"label":"` + label + `"}},"configured":{"targetKind":"sh_test rule"}}`} + for k := 1; k <= shards; k++ { + lines = append(lines, fmt.Sprintf(`{"id":{"testResult":{"label":"%s","shard":%d,"run":1,"attempt":1}}}`, label, k)) + } + path := filepath.Join(dir, fmt.Sprintf("bep%d.json", shards)) + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + return path + } + // A name ending in "~" is written as a skipped testcase. + writeXML := func(logs string, k int, names ...string) { + d := filepath.Join(logs, "pkg", "t", fmt.Sprintf("shard_%d_of_2", k)) + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + var b strings.Builder + b.WriteString(``) + for _, n := range names { + if skipped, ok := strings.CutSuffix(n, "~"); ok { + fmt.Fprintf(&b, ``, skipped) + continue + } + fmt.Fprintf(&b, ``, n) + } + b.WriteString(``) + if err := os.WriteFile(filepath.Join(d, "test.xml"), []byte(b.String()), 0o644); err != nil { + t.Fatal(err) + } + } + run := func(bep, logs string) (bool, string) { + cmd := exec.Command(python, script, "--bep", bep, "--testlogs", logs, "--suite", label, shard, "2") + out, err := cmd.CombinedOutput() + return err == nil, string(out) + } + cases := []struct { + name string + bepShard int + s1, s2 []string + pass bool + mention string + }{ + {"exact", 2, []string{"TestA", "TestB", "TestA/sub"}, []string{"TestC"}, true, ""}, + {"listed test missing from the binary", 2, []string{"TestA"}, []string{"TestC"}, false, "TestB is listed"}, + {"test in the wrong shard", 2, []string{"TestA", "TestB", "TestC"}, []string{}, false, "TestC"}, + {"unlisted test ran", 2, []string{"TestA", "TestB", "TestZ"}, []string{"TestC"}, false, "TestZ ran"}, + {"shard count differs", 1, []string{"TestA", "TestB"}, []string{"TestC"}, false, "want 2"}, + {"missing test.xml", 2, []string{"TestA", "TestB"}, nil, false, "missing"}, + // Review G5: a shard whose test.xml has no top-level test at all + // still reports every listed test missing (check_testcases.py also + // rejects it, but this check must not depend on that). + {"shard ran zero tests", 2, []string{}, []string{"TestC"}, false, "TestA is listed"}, + {"zero tests, only subtests", 2, []string{"TestA/sub"}, []string{"TestC"}, false, "TestB is listed"}, + // Review G3: some skips are fine; a shard of only skips is not. + {"some skipped", 2, []string{"TestA~", "TestB"}, []string{"TestC"}, true, ""}, + {"shard all skipped", 2, []string{"TestA~", "TestB~"}, []string{"TestC"}, false, "shard 1/2: every top-level test (2) was skipped"}, + } + for i, c := range cases { + t.Run(c.name, func(t *testing.T) { + logs := filepath.Join(dir, fmt.Sprintf("logs%d", i)) + writeXML(logs, 1, c.s1...) + if c.s2 != nil { + writeXML(logs, 2, c.s2...) + } + pass, out := run(writeBEP(c.bepShard), logs) + if pass != c.pass || (c.mention != "" && !strings.Contains(out, c.mention)) { + t.Errorf("pass = %v, want %v (mention %q):\n%s", pass, c.pass, c.mention, out) + } + }) + } + // --whole: an unsharded target must list a test and not only skips. + for i, c := range []struct { + names []string + pass bool + mention string + }{ + {[]string{"TestConformance"}, true, ""}, + {[]string{"TestConformance~"}, false, "was skipped"}, + {[]string{}, false, "lists no tests"}, + } { + logs := filepath.Join(dir, fmt.Sprintf("whole%d", i)) + d := filepath.Join(logs, "pkg", "w") + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + xml := `` + for _, n := range c.names { + if skipped, ok := strings.CutSuffix(n, "~"); ok { + xml += `` + } else { + xml += `` + } + } + xml += `` + if err := os.WriteFile(filepath.Join(d, "test.xml"), []byte(xml), 0o644); err != nil { + t.Fatal(err) + } + bep := filepath.Join(dir, fmt.Sprintf("whole%d.json", i)) + if err := os.WriteFile(bep, []byte(`{"id":{"targetConfigured":{"label":"//pkg:w"}},"configured":{"targetKind":"sh_test rule"}} +{"id":{"testResult":{"label":"//pkg:w","run":1,"attempt":1}}} +{"id":{"targetConfigured":{"label":"`+label+`"}},"configured":{"targetKind":"sh_test rule"}} +{"id":{"testResult":{"label":"`+label+`","shard":1}}} +{"id":{"testResult":{"label":"`+label+`","shard":2}}} +`), 0o644); err != nil { + t.Fatal(err) + } + writeXML(logs, 1, "TestA", "TestB") + writeXML(logs, 2, "TestC") + out, err := exec.Command(python, script, "--bep", bep, "--testlogs", logs, "--suite", label, shard, "2", "--whole", "//pkg:w").CombinedOutput() + if (err == nil) != c.pass || (c.mention != "" && !strings.Contains(string(out), c.mention)) { + t.Errorf("--whole %v: pass = %v, want %v (mention %q):\n%s", c.names, err == nil, c.pass, c.mention, out) + } + } + if out, err := exec.Command(python, script, "--bep", writeBEP(2), "--testlogs", filepath.Join(dir, "logs0"), "--suite", label, shard, "2", "--whole", "//pkg:absent").CombinedOutput(); err == nil { + t.Errorf("--whole for a target the BEP lacks passed:\n%s", out) + } + + // A failing shard script fails the check, even if what it listed + // matches; so does a missing one. + logs := filepath.Join(dir, "logs-bad") + writeXML(logs, 1, "TestA", "TestB") + writeXML(logs, 2, "TestC") + for _, c := range []struct { + script string + env []string + }{{shard, []string{"FAKE_SHARD_FAIL=1"}}, {"/nonexistent/shard.sh", nil}} { + cmd := exec.Command(python, script, "--bep", writeBEP(2), "--testlogs", logs, "--suite", label, c.script, "2") + cmd.Env = append(os.Environ(), c.env...) + if out, err := cmd.CombinedOutput(); err == nil || !strings.Contains(string(out), "failed") { + t.Errorf("shard script %s %v: check passed or did not say it failed:\n%s", c.script, c.env, out) + } + } +} + +// Review G1: the checker's input is the real shard scripts' list-only +// output. Every name they list, for every shard, must be a test go test +// runs (declared `func Name(t *testing.T)` in the package's _test.go files), +// or one check_shard_coverage.py drops as NOT_TESTS; and each NOT_TESTS name +// must really not be a test (TestMain takes *testing.M). Otherwise the +// checker reports a listed test that "did not run" on every real run. +func TestEmbeddedShardScriptsListOnlyRealTests(t *testing.T) { + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("scripts_test's runfiles hold neither the shard scripts' sources nor tools/bazel") + } + requireHostTool(t, "bash") + root := sourceRepoRoot(t) + m := regexp.MustCompile(`(?m)^NOT_TESTS = frozenset\(\{([^}]*)\}\)`).FindStringSubmatch(readPolicyFile(t, root, "tools/bazel/check_shard_coverage.py")) + if m == nil { + t.Fatal("tools/bazel/check_shard_coverage.py has no NOT_TESTS = frozenset({...})") + } + notTests := map[string]bool{} + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(m[1], -1) { + notTests[q[1]] = true + } + risk := readCIWorkflow(t, prRiskWorkflowName) + for _, c := range []struct{ job, script, pkg string }{ + {"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd"}, + {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt"}, + } { + var src strings.Builder + files, err := filepath.Glob(filepath.Join(root, c.pkg, "*_test.go")) + if err != nil || len(files) == 0 { + t.Fatalf("%s: no _test.go files (%v)", c.pkg, err) + } + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + src.Write(data) + src.WriteString("\n") + } + declared := map[string]bool{} + for _, d := range regexp.MustCompile(`(?m)^func (Test\w*)\(\w+ \*testing\.T\) \{`).FindAllStringSubmatch(src.String(), -1) { + declared[d[1]] = true + } + shards := len(risk.job(t, c.job).Strategy.Matrix.Shard) + listed := 0 + for k := 1; k <= shards; k++ { + cmd := exec.Command("bash", c.script, strconv.Itoa(k), strconv.Itoa(shards)) + cmd.Dir = root + cmd.Env = append(os.Environ(), "BEADS_TEST_SHARD_LIST_ONLY=1") + out, err := cmd.Output() + if err != nil { + t.Fatalf("%s %d %d: %v", c.script, k, shards, err) + } + for _, line := range strings.Split(string(out), "\n") { + name, ok := strings.CutPrefix(line, " ") + if !ok || !strings.HasPrefix(name, "Test") || strings.ContainsAny(name, " :") { + continue + } + listed++ + isTest := declared[name] + switch { + case notTests[name] && isTest: + t.Errorf("%s shard %d lists %s, which check_shard_coverage.py drops, but it is a real test", c.script, k, name) + case !notTests[name] && !isTest: + t.Errorf("%s shard %d lists %s, which is not a `func %s(t *testing.T)` test in %s: check_shard_coverage.py would report it missing on every run (add it to NOT_TESTS only if go test never runs it)", + c.script, k, name, name, c.pkg) + } + } + } + if listed < 50 { + t.Errorf("%s listed only %d tests over %d shards; did the list-only output format change?", c.script, listed, shards) + } + } + for name := range notTests { + if name != "TestMain" { + t.Errorf("NOT_TESTS has %s; only TestMain is never a test", name) + } + } +} + +// Review G3: since D2 step 1 the Bazel embedded lane is the tier's only +// pre-merge run on same-repo PRs, so nothing that reaches it may narrow it +// (select fewer tests, or turn them into skips) without a reviewed edit of +// this test. TestBazelEmbeddedJobMirrorsEmbeddedTier pins the command line +// and the --config=embedded lines; this covers everything else that applies +// to the lane: every other .bazelrc line of a config the lane uses (the +// unconfigured ones, remote-exec, which setup-bazel's rc enables, and any +// config those pull in), rc files that would be try-imported, the +// tools/bazel scripts every test runs under or through, the whole +// setup-bazel action, and the embedded-tagged targets' args and env. At run +// time, check_shard_coverage.py also fails a shard of only skips. +func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { + root := sourceRepoRoot(t) + rcNarrow := regexp.MustCompile(`test_filter|test_arg|-test\.|_filters\b|test_env=(BEADS_TEST|GO_TEST|TESTBRIDGE)|--config=|cache_test_results|run_under|flaky|runs_per_test|test_sharding_strategy|build_tests_only`) + const runUnder = "test --run_under=//tools/bazel:test_env" + wantImports := []string{"try-import %workspace%/.bazelrc.local", "try-import %workspace%/user.bazelrc"} + + type rcLine struct{ cmd, config, text string } + var lines []rcLine + var imports []string + sawRunUnder := false + for _, raw := range strings.Split(readPolicyFile(t, root, ".bazelrc"), "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + head, _, _ := strings.Cut(line, " ") + if head == "import" || head == "try-import" { + imports = append(imports, line) + continue + } + cmd, config, _ := strings.Cut(head, ":") + lines = append(lines, rcLine{cmd, config, line}) + } + if !reflect.DeepEqual(imports, wantImports) { + t.Errorf(".bazelrc imports %v, want exactly %v (an import can carry any flag into the lane)", imports, wantImports) + } + // The configs the lane uses: --config=embedded (pinned), remote-exec + // (setup-bazel's generated rc), the unconfigured lines, and anything they + // reference (which the check below then forbids anyway). + inUse := map[string]bool{"": true, "embedded": true, "remote-exec": true} + for changed := true; changed; { + changed = false + for _, l := range lines { + if !inUse[l.config] { + continue + } + for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(l.text, -1) { + if !inUse[m[1]] { + inUse[m[1]], changed = true, true + } + } + } + } + pinned := map[string]bool{} + for _, l := range bazelEmbeddedRCLines { + pinned[l] = true + } + for _, l := range lines { + if !inUse[l.config] || pinned[l.text] { + continue + } + if l.config == "embedded" { + t.Errorf(".bazelrc %q: not one of the pinned --config=embedded lines", l.text) + continue + } + if l.text == runUnder { + sawRunUnder = true + continue + } + if rcNarrow.MatchString(l.text) { + t.Errorf(".bazelrc %q applies to the embedded lane (config %q) and selects, narrows or re-runs tests", l.text, l.config) + } + } + if !sawRunUnder { + t.Errorf(".bazelrc lacks %q (the wrapper the narrowing checks below cover)", runUnder) + } + + // Committed rc files: only .bazelrc. .bazelrc.local and user.bazelrc are + // developer-local (gitignored) and would be try-imported into CI runs. + if os.Getenv("TEST_SRCDIR") == "" { + if git, err := exec.LookPath("git"); err == nil { + out, err := exec.Command(git, "-C", root, "ls-files").Output() + if err != nil { + t.Fatal(err) + } + for _, f := range strings.Split(strings.TrimSpace(string(out)), "\n") { + base := filepath.Base(f) + if strings.Contains(base, "bazelrc") && f != ".bazelrc" && f != setupBazelActionDir+"/write-bazelrc.sh" { + t.Errorf("committed rc file %s: .bazelrc's try-import would load it into every CI run", f) + } + } + } + } + + // The scripts every embedded test runs under or through, and the whole + // setup-bazel action (its generated rc applies to every command). + if os.Getenv("TEST_SRCDIR") == "" { + scriptNarrow := regexp.MustCompile(`-test\.(short|run|skip|list|bench)|BEADS_TEST_SKIP|BEADS_TEST_EMBEDDED_DOLT|TESTBRIDGE_TEST_ONLY|test_filter|test_arg|_filters\b|cache_test_results|flaky|runs_per_test|test_sharding_strategy`) + files, _ := filepath.Glob(filepath.Join(root, "tools", "bazel", "*.sh")) + action, _ := filepath.Glob(filepath.Join(root, setupBazelActionDir, "*")) + files = append(files, action...) + if len(files) < 5 { + t.Fatalf("found only %v", files) + } + for _, f := range files { + rel, _ := filepath.Rel(root, f) + data, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for i, line := range strings.Split(string(data), "\n") { + code := strings.TrimSpace(line) + if strings.HasPrefix(code, "#") { + continue + } + if scriptNarrow.MatchString(code) { + t.Errorf("%s:%d %q can select, skip or re-run the embedded lane's tests", rel, i+1, code) + } + for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(code, -1) { + if m[1] != "remote-exec" { + t.Errorf("%s:%d enables --config=%s for every command", rel, i+1, m[1]) + } + } + } + } + } + + // The embedded-tagged targets: exactly these, with exactly these args + // and env (the legacy jobs' flags; the shard scripts add the rest). + if os.Getenv("TEST_SRCDIR") != "" { + return // scripts_test's runfiles hold no other package's BUILD + } + type target struct { + args []string + env map[string]string + } + want := map[string]target{ + "//cmd/bd:bd_embedded_test": { + []string{"$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", "-test.timeout=19m"}, + map[string]string{"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)"}, + }, + "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": { + []string{"$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)"}, + map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, + "//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test": { + []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$", "-test.skip=^TestConformance$$/^Audit$$"}, + map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, + "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test": { + []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$/^Audit$$"}, + map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, + } + quoted := regexp.MustCompile(`"([^"]*)"`) + envPair := regexp.MustCompile(`"([^"]*)":\s*"([^"]*)"`) + nameRe := regexp.MustCompile(`(?m)^ name = "([^"]+)",$`) + tagsRe := regexp.MustCompile(`(?ms)^ tags = \[(.*?)\],$`) + envRe := regexp.MustCompile(`(?ms)^ env = \{(.*?)\},$`) + got := map[string]target{} + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + switch d.Name() { + case ".git", "node_modules", ".beads": + return filepath.SkipDir + } + return nil + } + if d.Type()&os.ModeSymlink != 0 || (d.Name() != "BUILD.bazel" && d.Name() != "BUILD") { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + pkg, _ := filepath.Rel(root, filepath.Dir(path)) + for _, rule := range bazelTopRules(string(data)) { + tags := tagsRe.FindStringSubmatch(rule) + if tags == nil || !strings.Contains(tags[1], `"embedded"`) { + continue + } + name := nameRe.FindStringSubmatch(rule) + if name == nil { + t.Errorf("%s: an embedded-tagged rule without a literal name:\n%s", pkg, rule) + continue + } + var tg target + for _, q := range quoted.FindAllStringSubmatch(bazelAttrBlock(rule, "args"), -1) { + tg.args = append(tg.args, q[1]) + } + tg.env = map[string]string{} + if e := envRe.FindStringSubmatch(rule); e != nil { + for _, p := range envPair.FindAllStringSubmatch(e[1], -1) { + tg.env[p[1]] = p[2] + } + } + if strings.Contains(rule, "args = select") || strings.Contains(rule, "env = select") || strings.Contains(rule, "env_inherit") { + t.Errorf("//%s:%s sets args/env indirectly; keep them literal so this check sees them", pkg, name[1]) + } + got["//"+filepath.ToSlash(pkg)+":"+name[1]] = tg + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("embedded-tagged targets' args/env changed:\ngot %v\nwant %v", got, want) + } +} + +// Review G4: bazel-embedded asks Bazel which test targets are flaky (a +// grep of BUILD files misses `flaky = _VAR` or a macro default) before it +// runs the tier, and fails on any. +func TestBazelEmbeddedQueriesFlakyTargets(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) + step := job.step(t, "No test target is marked flaky") + const want = `flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" +if [ -n "$flaky" ]; then + echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" + exit 1 +fi` + if strings.TrimSpace(step.Run) != want || step.If != "" || step.ContinueOnError != nil { + t.Errorf("flaky query step: if %q, continue-on-error %v, run:\n%s\nwant run:\n%s", step.If, step.ContinueOnError, step.Run, want) + } + if !(job.stepIndex(t, "Set up Bazel") < job.stepIndex(t, step.Name) && job.stepIndex(t, step.Name) < job.stepIndex(t, "bazel test //... --config=embedded")) { + t.Errorf("flaky query step must run after setup-bazel and before the tier") + } +} diff --git a/tools/bazel/check_shard_coverage.py b/tools/bazel/check_shard_coverage.py new file mode 100755 index 000000000..df78c5362 --- /dev/null +++ b/tools/bazel/check_shard_coverage.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Fail unless every Bazel shard ran exactly the tests its shard script lists. + +Usage: check_shard_coverage.py --bep [--testlogs DIR] + --suite LABEL SCRIPT SHARDS [--suite ...] + [--whole LABEL ...] + +The manifest-sharded targets (//cmd/bd:bd_embedded_test runs +.github/scripts/embedded-test-shard.sh, ...) discover their tests from +source with grep, but the Bazel test binary holds only the files in the +go_test's srcs. A discovered test the binary lacks matches nothing in the +shard's -test.run selector and passes silently, and check_testcases.py only +rejects a shard with no tests at all. + +For each --suite this runs SCRIPT k SHARDS with BEADS_TEST_SHARD_LIST_ONLY=1 +(from the repository root, like the CI jobs) for k = 1..SHARDS, and compares +the names it lists with the top-level names in the test.xml of +Bazel shard k of LABEL in this invocation's BEP. Any test listed but absent +(or present but not listed), a shard count other than SHARDS, a missing +test.xml or a failing script is an error. So is a shard (or any test.xml of +a --whole LABEL, an unsharded target such as the conformance partitions) +whose top-level tests are all skipped: an injected -test.short, +BEADS_TEST_SKIP or a lost BEADS_TEST_EMBEDDED_DOLT=1 turns the tier into +t.Skip calls, which still list every test. It needs test.xml locally and +-test.v in it, as --config=embedded sets. + +Exit status: 0 if every shard matches, 1 otherwise. +""" + +import argparse +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from equivalence import read_bep, testlog_xmls # noqa: E402 + +LISTED = re.compile(r"^ (Test[A-Za-z0-9_]*)$") + +# Names the shard scripts' `grep '^func Test'` discovery lists that go test +# never runs as a test: TestMain(m *testing.M) is the package's test entry +# point (internal/storage/embeddeddolt/test_fixture_test.go), never a +# . The legacy jobs' -test.run selector harmlessly matches nothing +# for it. Filtered here rather than in the shard scripts so the legacy jobs' +# selection stays byte-for-byte what it was. scripts/ +# pr_risk_bazel_coverage_test.go runs both real scripts in list-only mode and +# requires every other listed name to be a `func Name(t *testing.T)`. +NOT_TESTS = frozenset({"TestMain"}) + + +def listed_tests(script, shard, shards): + """Return the set of tests SCRIPT assigns to shard SHARD of SHARDS.""" + env = dict(os.environ, BEADS_TEST_SHARD_LIST_ONLY="1") + out = subprocess.run( + ["bash", script, str(shard), str(shards)], + env=env, check=True, capture_output=True, text=True, + ).stdout + return {m.group(1) for m in map(LISTED.match, out.splitlines()) if m} - NOT_TESTS + + +def ran_tests(xml_path): + """Return (set of top-level Go tests, set of those skipped) in a test.xml.""" + names, skipped = set(), set() + for tc in ET.parse(xml_path).getroot().iter("testcase"): + name = tc.get("name", "") + if name.startswith("Test") and "/" not in name: + names.add(name) + if tc.find("skipped") is not None: + skipped.add(name) + return names, skipped + + +def all_skipped_problem(where, got, skipped): + if got and got == skipped: + return f"{where}: every top-level test ({len(got)}) was skipped" + return None + + +def check(tested, testlogs, suites, lister=listed_tests, whole=()): + """Return ([summary lines], [problems]) for suites of (label, script, shards) + and whole (unsharded or sharded) labels that must not be all-skipped.""" + lines, problems = [], [] + for label in whole: + if label not in tested: + problems.append(f"{label}: --whole, but the BEP has no result for it") + continue + for xml_path in testlog_xmls(testlogs, label, tested[label]): + rel = os.path.relpath(xml_path, testlogs) + if not os.path.exists(xml_path): + problems.append(f"{label}: {rel} missing (was test.xml downloaded?)") + continue + try: + got, skipped = ran_tests(xml_path) + except ET.ParseError as e: + problems.append(f"{label}: cannot parse {rel}: {e}") + continue + if not got: + problems.append(f"{label}: {rel} lists no tests") + p = all_skipped_problem(f"{label}: {rel}", got, skipped) + if p: + problems.append(p) + lines.append(f"{label}: not all skipped") + for label, script, shards in suites: + if tested.get(label) != shards: + problems.append(f"{label}: the BEP has {tested.get(label, 0)} shard(s), want {shards} ({script})") + continue + total = 0 + for k, xml_path in enumerate(testlog_xmls(testlogs, label, shards), start=1): + try: + want = lister(script, k, shards) + except (OSError, subprocess.CalledProcessError) as e: + problems.append(f"{label}: {script} {k} {shards} failed: {e}") + continue + rel = os.path.relpath(xml_path, testlogs) + if not os.path.exists(xml_path): + problems.append(f"{label}: {rel} missing (was test.xml downloaded?)") + continue + try: + got, skipped = ran_tests(xml_path) + except ET.ParseError as e: + problems.append(f"{label}: cannot parse {rel}: {e}") + continue + total += len(want) + p = all_skipped_problem(f"{label} shard {k}/{shards}", got, skipped) + if p: + problems.append(p) + for name in sorted(want - got): + problems.append(f"{label} shard {k}/{shards}: {name} is listed by {script} but did not run " + f"(not in the target's srcs?)") + for name in sorted(got - want): + problems.append(f"{label} shard {k}/{shards}: {name} ran but {script} does not list it there") + lines.append(f"{label}: {shards} shards, {total} listed tests") + return lines, problems + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") + ap.add_argument("--testlogs", default=None, help="default: from the BEP, else ./bazel-testlogs") + ap.add_argument("--suite", nargs=3, action="append", required=True, metavar=("LABEL", "SCRIPT", "SHARDS"), + help="a manifest-sharded target, its shard script and shard count (repeatable)") + ap.add_argument("--whole", action="append", default=[], metavar="LABEL", + help="a target none of whose test.xml may be all skipped (repeatable)") + args = ap.parse_args(argv) + + suites = [] + for label, script, shards in args.suite: + if not shards.isdigit() or int(shards) < 1: + ap.error(f"--suite {label}: SHARDS must be a positive integer, got {shards!r}") + suites.append((label, script, int(shards))) + tested, _, bep_testlogs = read_bep(args.bep) + testlogs = args.testlogs or bep_testlogs or "bazel-testlogs" + lines, problems = check(tested, testlogs, suites, whole=args.whole) + for line in lines: + print(line) + for p in problems: + print(f"FAIL: {p}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main())