Compare commits
15 commits
main
...
ci/d2-reti
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
75cdf5d40c | ||
|
|
36d152a135 | ||
|
|
b92a8776d1 | ||
|
|
176c522362 | ||
|
|
3a1d1d2a7b | ||
|
|
f75f8dee9f | ||
|
|
89ddc834ef | ||
|
|
8ee6838e32 | ||
|
|
3c672c1d2c | ||
|
|
d0b0b09041 | ||
|
|
281ef603e8 | ||
|
|
f4df4e4bf3 | ||
|
|
567bd8483b | ||
|
|
ff49befd87 | ||
|
|
e86f39bbdf |
10 changed files with 1862 additions and 31 deletions
6
.bazelrc
6
.bazelrc
|
|
@ -99,6 +99,12 @@ test:embedded --test_timeout=-1,-1,-1,1200
|
|||
test:embedded --test_arg=-test.parallel=4
|
||||
test:embedded --test_env=GO_TEST_WRAP_TESTV=1
|
||||
test:embedded --remote_download_regex=.*/test\.xml$
|
||||
# Since D2 step 1 this lane is the tier's only pre-merge run on same-repo PRs
|
||||
# (pr-risk.yml's legacy jobs stand down), and the legacy jobs ran every test
|
||||
# every time (-test.count=1). A cached PASS would let a stale or poisoned
|
||||
# farm action-cache entry, or an unreproduced -race finding, stand in for a
|
||||
# run: always execute, at farm (not runner) cost.
|
||||
test:embedded --nocache_test_results
|
||||
|
||||
# Dolt-server lane: the same jobs as the docker lane ("Test (storage domain +
|
||||
# uow)" and "Contract corpus"), but every dolt sql-server is a child of the
|
||||
|
|
|
|||
41
.github/workflows/bazel.yml
vendored
41
.github/workflows/bazel.yml
vendored
|
|
@ -5,7 +5,13 @@
|
|||
# and server-Dolt storage tiers, main.yml's integration jobs).
|
||||
#
|
||||
# PRs and merge groups run it through pr.yml's `bazel` job (workflow_call),
|
||||
# in the same run as the legacy jobs it mirrors, which stay required too.
|
||||
# in the same run as the legacy jobs it mirrors, which stay required too,
|
||||
# except pr-risk.yml's embedded-Dolt test jobs: while the committed
|
||||
# BAZEL_RETIRES_LEGACY_EMBEDDED flag is "true", on a same-repo,
|
||||
# non-Dependabot pull_request (whatever the mode), pr-risk.yml's bazel-embedded-coverage job stands them down and
|
||||
# bazel-embedded alone runs the tier; pr.yml's ci-gate then requires that
|
||||
# lane to have run remotely and passed, so modes skip and local are red there
|
||||
# (scripts/pr_risk_bazel_coverage_test.go).
|
||||
# This file has no pull_request or merge_group trigger of its own, so a PR
|
||||
# runs the lane once. Every lane reports its job.status as a workflow_call
|
||||
# output (named after the job) from an always() last step. pr.yml's ci-gate
|
||||
|
|
@ -659,6 +665,9 @@ jobs:
|
|||
# processes (~7-10k test-seconds) on one runner, an hour or more per run
|
||||
# on the pool whose job cap this track exists to relieve, and it proves
|
||||
# nothing pr-risk.yml's embedded jobs (which do run for forks) do not.
|
||||
# On same-repo pull_requests those jobs stand down (pr-risk.yml's
|
||||
# bazel-embedded-coverage), so this lane is then the tier's only PR run
|
||||
# and pr.yml's gate requires it to have run here and passed.
|
||||
# The runs-on expression is the shared one and always picks the
|
||||
# Blacksmith runner under this if.
|
||||
needs: rbe
|
||||
|
|
@ -687,6 +696,18 @@ jobs:
|
|||
RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }}
|
||||
RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }}
|
||||
|
||||
# Bazel retries a flaky = True test (up to three attempts) and reports a
|
||||
# later pass as PASSED. The policy test greps BUILD files for it; this
|
||||
# asks Bazel itself, so a macro or a variable cannot hide one. Cheap:
|
||||
# loading only, with the repository cache the test step needs anyway.
|
||||
- name: No test target is marked flaky
|
||||
run: |
|
||||
flaky="$(bazel query 'attr(flaky, 1, tests(//...))')"
|
||||
if [ -n "$flaky" ]; then
|
||||
echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --build_event_json_file is a command option, so it goes after `test`.
|
||||
- name: bazel test //... --config=embedded
|
||||
id: test
|
||||
|
|
@ -708,6 +729,24 @@ jobs:
|
|||
if: ${{ always() && steps.test.outcome != 'skipped' }}
|
||||
run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json"
|
||||
|
||||
# Since D2 step 1 this lane is the tier's only pre-merge run on
|
||||
# same-repo PRs. The shard scripts discover tests from source, but each
|
||||
# Bazel binary holds only its target's srcs: a listed test the binary
|
||||
# lacks matches nothing and passes silently. Fail unless every Bazel
|
||||
# shard ran exactly the tests its script lists (list-only mode, cheap),
|
||||
# and unless every shard and conformance partition ran at least one
|
||||
# top-level test that was not skipped (an injected -test.short or
|
||||
# BEADS_TEST_SKIP turns the tier into t.Skip calls).
|
||||
- name: Every listed test ran in its shard
|
||||
if: ${{ always() && steps.test.outcome != 'skipped' }}
|
||||
run: >-
|
||||
python3 tools/bazel/check_shard_coverage.py
|
||||
--bep "$RUNNER_TEMP/bazel-bep.json"
|
||||
--suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 20
|
||||
--suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 5
|
||||
--whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test
|
||||
--whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test
|
||||
|
||||
- name: Upload test logs
|
||||
if: ${{ failure() && steps.test.outcome != 'skipped' }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
|
|
|
|||
82
.github/workflows/pr-risk.yml
vendored
82
.github/workflows/pr-risk.yml
vendored
|
|
@ -12,6 +12,15 @@ concurrency:
|
|||
env:
|
||||
BD_DISABLE_METRICS: "1"
|
||||
BD_DISABLE_EVENT_FLUSH: "1"
|
||||
# D2 step 1: "true" retires PR Risk's legacy embedded-Dolt test jobs on
|
||||
# same-repo, non-Dependabot pull_requests; pr.yml's
|
||||
# gated Bazel lane is then the tier's only pre-merge run, and pr.yml's
|
||||
# gate goes red unless that lane ran remotely and passed. Committed, not a
|
||||
# repo variable, so a re-run can never change it; must equal pr.yml's /
|
||||
# pr-risk.yml's copy (scripts/pr_risk_bazel_coverage_test.go). Revert:
|
||||
# commit "false" in both workflows (in-flight PRs pick it up on their next
|
||||
# push or merge of main).
|
||||
BAZEL_RETIRES_LEGACY_EMBEDDED: "true"
|
||||
|
||||
jobs:
|
||||
detect-ci-tier:
|
||||
|
|
@ -55,6 +64,52 @@ jobs:
|
|||
printf '%s\n' "$selection" | sed 's/^/- `/' | sed 's/$/`/'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# D2 step 1: whether the legacy test-embedded-* jobs below stand down
|
||||
# because pr.yml's gated Bazel `embedded Dolt tier` lane is this PR's run of
|
||||
# the tier. True only with BAZEL_RETIRES_LEGACY_EMBEDDED, on a same-repo (not
|
||||
# fork) pull_request whose actor is not Dependabot. pr.yml runs this same
|
||||
# job (policy-tested identical) and its gate goes red unless the lane then
|
||||
# ran remotely and passed, so neither the kill switch (mode skip) nor a
|
||||
# missing executor secret (mode local) can silently drop the tier. Forks
|
||||
# and Dependabot (no secrets), merge_group and every other event keep the
|
||||
# legacy tier. The step reads no secret or variable and runs no repository
|
||||
# code.
|
||||
bazel-embedded-coverage:
|
||||
name: Bazel embedded coverage
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
covered: ${{ steps.decide.outputs.covered }}
|
||||
steps:
|
||||
# GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's
|
||||
# expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS
|
||||
# and not the executor secret (either, evaluated per run, would let one
|
||||
# workflow's re-run disagree with the other's). A same-repo PR whose run
|
||||
# lacks the secret is covered anyway, so pr.yml's gate goes red (mode
|
||||
# local) instead of the tier silently moving back. Dependabot runs get
|
||||
# no Actions secrets by design and keep the legacy tier; github.actor
|
||||
# (unlike triggering_actor) stays the PR's actor on a re-run.
|
||||
- name: Decide whether the Bazel lane covers the embedded tier
|
||||
id: decide
|
||||
env:
|
||||
RETIRED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }}
|
||||
PULL_REQUEST: ${{ github.event_name == 'pull_request' }}
|
||||
FORK: ${{ github.event.pull_request.head.repo.fork == true }}
|
||||
DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }}
|
||||
run: |
|
||||
for v in RETIRED PULL_REQUEST FORK DEPENDABOT; do
|
||||
case "${!v}" in
|
||||
true|false) ;;
|
||||
*) echo "::error::$v is '${!v}', want true or false"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
covered=false
|
||||
if [ "$RETIRED" = true ] && [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then
|
||||
covered=true
|
||||
fi
|
||||
echo "covered=$covered" >> "$GITHUB_OUTPUT"
|
||||
echo "Legacy embedded Dolt tier retired in favour of the Bazel lane: **$covered** (retired: $RETIRED, pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT)" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# End-to-end coverage for proxied-server CLI subcommands, sharded across the
|
||||
# CI matrix. Each sub-test drives the bd CLI subprocess against a real dolt
|
||||
# sql-server started via testcontainers by the shared proxied harness
|
||||
|
|
@ -119,6 +174,11 @@ jobs:
|
|||
# Split into: build → storage tests + parallel cmd test shards.
|
||||
# cmd/bd embedded tests use a committed shard manifest; newly-added tests
|
||||
# still auto-distribute via hash fallback until the manifest is updated.
|
||||
#
|
||||
# The storage, conformance and cmd test jobs stand down when
|
||||
# bazel-embedded-coverage says pr.yml's gated Bazel `embedded Dolt tier`
|
||||
# lane runs them instead; build-embedded keeps running, because its
|
||||
# artifact also feeds the proxied and server Dolt jobs.
|
||||
|
||||
build-embedded:
|
||||
name: Build (Embedded Dolt)
|
||||
|
|
@ -202,8 +262,8 @@ jobs:
|
|||
|
||||
test-embedded-storage:
|
||||
name: Test (Embedded Dolt Storage ${{ matrix.shard }}/${{ strategy.job-total }})
|
||||
needs: [detect-ci-tier, build-embedded]
|
||||
if: needs.detect-ci-tier.outputs.full_embedded == 'true'
|
||||
needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded]
|
||||
if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
strategy:
|
||||
|
|
@ -240,8 +300,8 @@ jobs:
|
|||
|
||||
test-embedded-conformance:
|
||||
name: Test (Embedded Dolt Conformance - ${{ matrix.partition }})
|
||||
needs: [detect-ci-tier, build-embedded]
|
||||
if: needs.detect-ci-tier.outputs.full_embedded == 'true'
|
||||
needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded]
|
||||
if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 35
|
||||
strategy:
|
||||
|
|
@ -391,8 +451,8 @@ jobs:
|
|||
|
||||
test-embedded-cmd:
|
||||
name: Test (Embedded Dolt Cmd ${{ matrix.shard }}/${{ strategy.job-total }})
|
||||
needs: [detect-ci-tier, build-embedded]
|
||||
if: needs.detect-ci-tier.outputs.full_embedded == 'true'
|
||||
needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded]
|
||||
if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
|
@ -457,6 +517,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- detect-ci-tier
|
||||
- bazel-embedded-coverage
|
||||
- build-embedded
|
||||
- test-embedded-storage
|
||||
- test-embedded-conformance
|
||||
|
|
@ -474,6 +535,7 @@ jobs:
|
|||
CI_GATE_NAME: PR risk gate
|
||||
CI_GATE_REQUIRED: >-
|
||||
DETECT_CI_TIER
|
||||
BAZEL_EMBEDDED_COVERAGE
|
||||
BUILD_EMBEDDED
|
||||
TEST_EMBEDDED_STORAGE
|
||||
TEST_EMBEDDED_CONFORMANCE
|
||||
|
|
@ -483,7 +545,9 @@ jobs:
|
|||
TEST_PROXIED_CMD
|
||||
TEST_NIX
|
||||
FULL_EMBEDDED: ${{ needs.detect-ci-tier.outputs.full_embedded }}
|
||||
BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-embedded-coverage.outputs.covered }}
|
||||
DETECT_CI_TIER: ${{ needs.detect-ci-tier.result }}
|
||||
BAZEL_EMBEDDED_COVERAGE: ${{ needs.bazel-embedded-coverage.result }}
|
||||
BUILD_EMBEDDED: ${{ needs.build-embedded.result }}
|
||||
TEST_EMBEDDED_STORAGE: ${{ needs.test-embedded-storage.result }}
|
||||
TEST_EMBEDDED_CONFORMANCE: ${{ needs.test-embedded-conformance.result }}
|
||||
|
|
@ -497,5 +561,11 @@ jobs:
|
|||
if [[ "$FULL_EMBEDDED" != "true" ]]; then
|
||||
skipped_ok="BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD"
|
||||
fi
|
||||
# The legacy embedded test jobs stand down only where pr.yml's gated
|
||||
# Bazel lane runs the tier (bazel-embedded-coverage); any other skip
|
||||
# of them, or a failed or missing decision, fails the gate.
|
||||
if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]]; then
|
||||
skipped_ok="$skipped_ok TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_EMBEDDED_CMD"
|
||||
fi
|
||||
export CI_GATE_SKIPPED_OK="$skipped_ok"
|
||||
bash .github/scripts/ci-gate.sh
|
||||
|
|
|
|||
72
.github/workflows/pr.yml
vendored
72
.github/workflows/pr.yml
vendored
|
|
@ -15,6 +15,15 @@ concurrency:
|
|||
env:
|
||||
BD_DISABLE_METRICS: "1"
|
||||
BD_DISABLE_EVENT_FLUSH: "1"
|
||||
# D2 step 1: "true" retires PR Risk's legacy embedded-Dolt test jobs on
|
||||
# same-repo, non-Dependabot pull_requests; pr.yml's
|
||||
# gated Bazel lane is then the tier's only pre-merge run, and pr.yml's
|
||||
# gate goes red unless that lane ran remotely and passed. Committed, not a
|
||||
# repo variable, so a re-run can never change it; must equal pr.yml's /
|
||||
# pr-risk.yml's copy (scripts/pr_risk_bazel_coverage_test.go). Revert:
|
||||
# commit "false" in both workflows (in-flight PRs pick it up on their next
|
||||
# push or merge of main).
|
||||
BAZEL_RETIRES_LEGACY_EMBEDDED: "true"
|
||||
|
||||
jobs:
|
||||
|
||||
|
|
@ -1760,7 +1769,11 @@ jobs:
|
|||
# The Bazel lane (bazel.yml), gated alongside the legacy jobs it mirrors
|
||||
# (PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and
|
||||
# contract corpus, and pr-risk.yml's embedded-Dolt, proxied-server and
|
||||
# server-Dolt storage tiers), all of which stay required. bazel.yml has no pull_request/merge_group trigger of its own:
|
||||
# server-Dolt storage tiers), all of which stay required, except that
|
||||
# pr-risk.yml's embedded-Dolt test jobs stand down on same-repo PRs with
|
||||
# a non-Dependabot actor while BAZEL_RETIRES_LEGACY_EMBEDDED is "true" (its
|
||||
# bazel-embedded-coverage job; D2 step 1), and ci-gate then requires this
|
||||
# call's embedded lane to have run remotely and passed. bazel.yml has no pull_request/merge_group trigger of its own:
|
||||
# this call is its only PR run. It gets the four RBE secrets and nothing
|
||||
# else; its rbe job decides remote, local (forks, Dependabot: no
|
||||
# remote-only lanes) or skip, and exports that for ci-gate. Its
|
||||
|
|
@ -1783,6 +1796,46 @@ jobs:
|
|||
build-artifact-name: bazel-ci-build-artifacts
|
||||
integration: "on"
|
||||
|
||||
# D2 step 1: pr-risk.yml's bazel-embedded-coverage job, verbatim (policy-
|
||||
# tested identical). Where it says covered, PR Risk skips its legacy
|
||||
# embedded test jobs, so ci-gate requires the Bazel embedded lane to have
|
||||
# run remotely and passed in this run.
|
||||
bazel-embedded-coverage:
|
||||
name: Bazel embedded coverage
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
covered: ${{ steps.decide.outputs.covered }}
|
||||
steps:
|
||||
# GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's
|
||||
# expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS
|
||||
# and not the executor secret (either, evaluated per run, would let one
|
||||
# workflow's re-run disagree with the other's). A same-repo PR whose run
|
||||
# lacks the secret is covered anyway, so pr.yml's gate goes red (mode
|
||||
# local) instead of the tier silently moving back. Dependabot runs get
|
||||
# no Actions secrets by design and keep the legacy tier; github.actor
|
||||
# (unlike triggering_actor) stays the PR's actor on a re-run.
|
||||
- name: Decide whether the Bazel lane covers the embedded tier
|
||||
id: decide
|
||||
env:
|
||||
RETIRED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }}
|
||||
PULL_REQUEST: ${{ github.event_name == 'pull_request' }}
|
||||
FORK: ${{ github.event.pull_request.head.repo.fork == true }}
|
||||
DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }}
|
||||
run: |
|
||||
for v in RETIRED PULL_REQUEST FORK DEPENDABOT; do
|
||||
case "${!v}" in
|
||||
true|false) ;;
|
||||
*) echo "::error::$v is '${!v}', want true or false"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
covered=false
|
||||
if [ "$RETIRED" = true ] && [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then
|
||||
covered=true
|
||||
fi
|
||||
echo "covered=$covered" >> "$GITHUB_OUTPUT"
|
||||
echo "Legacy embedded Dolt tier retired in favour of the Bazel lane: **$covered** (retired: $RETIRED, pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT)" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
ci-gate:
|
||||
name: CI Gate / Required
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -1811,6 +1864,7 @@ jobs:
|
|||
- lint
|
||||
- windows-make-shell
|
||||
- bazel
|
||||
- bazel-embedded-coverage
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
|
||||
|
|
@ -1850,6 +1904,8 @@ jobs:
|
|||
BAZEL_DOLTSERVER
|
||||
BAZEL_PROXIED
|
||||
BAZEL_SERVER_STORAGE
|
||||
BAZEL_EMBEDDED_COVERAGE
|
||||
BAZEL_EMBEDDED_RETIRED
|
||||
BUILD_ARTIFACTS: ${{ needs.build-artifacts.result }}
|
||||
CHECK_BUILD_TAGS: ${{ needs.check-build-tags.result }}
|
||||
CHECK_CMD_BD_PUREGEO_TESTS: ${{ needs.check-cmd-bd-puregeo-tests.result }}
|
||||
|
|
@ -1889,6 +1945,10 @@ jobs:
|
|||
BAZEL_CALL: ${{ needs.bazel.result }}
|
||||
BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }}
|
||||
BAZEL_RBE_ENABLED: ${{ needs.bazel.outputs.rbe-enabled }}
|
||||
# D2 step 1: whether PR Risk retired its legacy embedded tier for
|
||||
# this PR (BAZEL_EMBEDDED_RETIRED below).
|
||||
BAZEL_EMBEDDED_COVERAGE: ${{ needs.bazel-embedded-coverage.result }}
|
||||
BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-embedded-coverage.outputs.covered }}
|
||||
run: |
|
||||
skipped_ok=""
|
||||
if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then
|
||||
|
|
@ -1901,5 +1961,15 @@ jobs:
|
|||
skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)"
|
||||
BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)"
|
||||
export BAZEL
|
||||
# Where PR Risk's legacy embedded tier is retired, this run's Bazel
|
||||
# embedded lane is the tier's only pre-merge run: it must have run
|
||||
# remotely and passed. Mode skip (the farm switch off), or any
|
||||
# other result, is red whatever bazel-gate.sh allows.
|
||||
BAZEL_EMBEDDED_RETIRED=success
|
||||
if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]] && { [[ "$BAZEL_RBE_MODE" != "remote" ]] || [[ "$BAZEL_EMBEDDED" != "success" ]]; }; then
|
||||
BAZEL_EMBEDDED_RETIRED="not-run-remotely"
|
||||
echo "::error::PR Risk's legacy embedded Dolt tier is retired (BAZEL_RETIRES_LEGACY_EMBEDDED) and the Bazel embedded lane did not run remotely and pass (mode '${BAZEL_RBE_MODE:-none}', result '${BAZEL_EMBEDDED:-none}'). If remote execution is off on purpose, commit BAZEL_RETIRES_LEGACY_EMBEDDED: \"false\" in pr.yml and pr-risk.yml and push this PR again."
|
||||
fi
|
||||
export BAZEL_EMBEDDED_RETIRED
|
||||
export CI_GATE_SKIPPED_OK="$skipped_ok"
|
||||
bash .github/scripts/ci-gate.sh
|
||||
|
|
|
|||
|
|
@ -49,7 +49,9 @@ Current PR-related workflow names:
|
|||
call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`,
|
||||
`BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and
|
||||
`BAZEL_SERVER_STORAGE`. The legacy jobs these mirror stay required in
|
||||
`pr.yml` and `pr-risk.yml`.
|
||||
`pr.yml` and `pr-risk.yml`, except where `BAZEL_EMBEDDED` runs in their
|
||||
place (see
|
||||
[Legacy Embedded Tier Retirement](#legacy-embedded-tier-retirement-d2-step-1)).
|
||||
`.github/scripts/bazel-gate.sh` reads the exported mode, never the variable
|
||||
or the fork flag: mode `skip` allows every Bazel id to skip, mode `local`
|
||||
allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`,
|
||||
|
|
@ -77,10 +79,16 @@ Current PR-related workflow names:
|
|||
red, and autofix only patches packages the PR itself changed), the RBE
|
||||
farm is down, or the beads CI RBE client certificate expires (about
|
||||
2027-09-27; a partial secret set also fails every same-repo run), fix
|
||||
`main` (`make bazel-sync`) or renew the secrets, or unset the
|
||||
`RBE_WEST_WORKERS` repo variable: same-repo runs then take mode `skip`,
|
||||
which the gate accepts (fork PRs still run locally, so drift on `main`
|
||||
still reaches them).
|
||||
`main` (`make bazel-sync`) or renew the secrets. To turn remote execution
|
||||
off instead, first commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"` in both
|
||||
`pr.yml` and `pr-risk.yml` (see
|
||||
[Legacy Embedded Tier Retirement](#legacy-embedded-tier-retirement-d2-step-1)),
|
||||
then unset the `RBE_WEST_WORKERS` repo variable: same-repo runs then take
|
||||
mode `skip`, which the gate accepts (fork PRs still run locally, so drift
|
||||
on `main` still reaches them). Unsetting the variable while the flag is
|
||||
still `"true"` turns `CI Gate / Required` red on every same-repo PR
|
||||
(`BAZEL_EMBEDDED_RETIRED`), because PR Risk no longer runs the legacy
|
||||
embedded tier for them.
|
||||
- `.github/workflows/bazel-farm.yml`: `Bazel Farm (trusted forks)`
|
||||
Runs on `pull_request_target` for fork PRs to `main` whose author and
|
||||
triggering user are on `.github/bazel-farm-allowlist.txt`, and calls
|
||||
|
|
@ -89,8 +97,9 @@ Current PR-related workflow names:
|
|||
[Trusted-Author Fork PRs](#trusted-author-fork-prs-bazel-farm).
|
||||
- `.github/workflows/pr-risk.yml`: `PR Risk`
|
||||
Runs on `pull_request` and `merge_group`. Contains embedded Dolt risk
|
||||
detection, embedded build/test shards, the Nix flake smoke check, and the
|
||||
risk aggregate gate `PR Risk / PR Risk Gate / Required`.
|
||||
detection, the `bazel-embedded-coverage` decision, embedded build/test
|
||||
shards, the proxied and server Dolt shards, the Nix flake smoke check, and
|
||||
the risk aggregate gate `PR Risk / PR Risk Gate / Required`.
|
||||
- `.github/workflows/main.yml`: `Main`
|
||||
Runs on pushes to `main`. Contains the main branch health checks, package
|
||||
gates, platform smoke/short coverage, embedded Dolt coverage, and promoted
|
||||
|
|
@ -121,8 +130,12 @@ the `bazel` call) and `PR Risk Gate / Required` (`pr-risk.yml`). PR Risk's
|
|||
gate job was renamed from `CI Gate / Required` to `PR Risk Gate / Required`
|
||||
(#6939), so the two required contexts are distinct; before that both
|
||||
workflows reported the same check name. Organization admins and one team
|
||||
may bypass it. There is no merge queue, so the checks run on
|
||||
`pull_request`.
|
||||
may bypass it. There is no merge queue (no `merge_queue` rule, and no
|
||||
`merge_group` run has ever happened), so the checks run on `pull_request`
|
||||
only, and the ruleset does not require branches to be up to date
|
||||
(strict off). The ruleset applies to the default branch only: PRs into
|
||||
`release/**` run both workflows (and the Bazel lane), but neither gate is
|
||||
enforced there.
|
||||
|
||||
## Trusted-Author Fork PRs (Bazel Farm)
|
||||
|
||||
|
|
@ -420,7 +433,8 @@ would only be needed if maintainers still want exactly one required check.
|
|||
- Baseline aggregate candidate: `PR / CI Gate / Required`
|
||||
- Risk aggregate candidate: `PR Risk / PR Risk Gate / Required`
|
||||
- Source: GitHub Actions
|
||||
- Required on: pull requests and merge queue groups targeting `main`
|
||||
- Required on: pull requests (and merge queue groups, if a queue is ever
|
||||
added) targeting `main`; not on `release/**`
|
||||
|
||||
Do not require these existing check names directly:
|
||||
|
||||
|
|
@ -433,8 +447,10 @@ Do not require these existing check names directly:
|
|||
- `Test (ubuntu-latest)`
|
||||
- `Test (macos-latest)`
|
||||
- `Test (storage domain + uow)`
|
||||
- `Bazel embedded coverage`
|
||||
- `Build (Embedded Dolt)`
|
||||
- `Test (Embedded Dolt Storage 1/5)` through `Test (Embedded Dolt Storage 5/5)`
|
||||
- `Test (Embedded Dolt Conformance - core)` and `- audit`
|
||||
- `Test (Embedded Dolt Cmd 1/20)` through `Test (Embedded Dolt Cmd 20/20)`
|
||||
- `Test (Windows - smoke)`
|
||||
- `Check formatting`
|
||||
|
|
@ -554,7 +570,8 @@ intentionally not kept in lockstep with later leaf additions and renames; use
|
|||
<!-- markdownlint-enable MD013 -->
|
||||
|
||||
`.github/workflows/pr-risk.yml` has a companion aggregate for `detect-ci-tier`,
|
||||
`build-embedded`, `test-embedded-storage`, `test-embedded-cmd`, and `test-nix`.
|
||||
`bazel-embedded-coverage`, `build-embedded`, the embedded, proxied and server
|
||||
Dolt test jobs, and `test-nix`.
|
||||
|
||||
`.github/scripts/ci-gate.sh` is a small shell evaluator. It fails on any
|
||||
`failure` or `cancelled` result. It accepts `skipped` only for jobs that are
|
||||
|
|
@ -562,8 +579,15 @@ intentionally absent for that event or risk tier:
|
|||
|
||||
- `CHECK_NO_BEADS_CHANGES=skipped` is acceptable on `merge_group` because the
|
||||
job is PR-only.
|
||||
- In the risk aggregate, `BUILD_EMBEDDED`, `TEST_EMBEDDED_STORAGE`, and
|
||||
`TEST_EMBEDDED_CMD` may be `skipped` only when `FULL_EMBEDDED != true`.
|
||||
- In the risk aggregate, `BUILD_EMBEDDED` and the embedded, proxied and server
|
||||
Dolt test ids may be `skipped` when `FULL_EMBEDDED != true`.
|
||||
- In the risk aggregate, `TEST_EMBEDDED_STORAGE`, `TEST_EMBEDDED_CONFORMANCE`
|
||||
and `TEST_EMBEDDED_CMD` may also be `skipped` when
|
||||
`bazel-embedded-coverage` reported `covered=true`. `BAZEL_EMBEDDED_COVERAGE`
|
||||
(that job's result) must be `success`.
|
||||
- In the baseline aggregate, `BAZEL_EMBEDDED_COVERAGE` must be `success` and
|
||||
`BAZEL_EMBEDDED_RETIRED` is red when `covered=true` but the Bazel embedded
|
||||
lane did not run remotely and pass.
|
||||
- All baseline jobs must be `success`.
|
||||
|
||||
This keeps branch protection pointed at stable aggregate jobs while preserving
|
||||
|
|
@ -614,13 +638,144 @@ listed paths can be blocked waiting for a check that GitHub never creates.
|
|||
The current embedded Dolt topology already fits the required-check model:
|
||||
|
||||
- `detect-ci-tier` always runs.
|
||||
- `build-embedded`, `test-embedded-storage`, and `test-embedded-cmd` use
|
||||
job-level `if`.
|
||||
- `build-embedded`, `test-embedded-storage`, `test-embedded-conformance` and
|
||||
`test-embedded-cmd` use job-level `if`; the three test jobs also stand down
|
||||
where the Bazel lane covers them (next section).
|
||||
- `.github/scripts/ci-embedded-tier.sh` runs full embedded coverage for
|
||||
`push`, `merge_group`, unavailable PR diff bounds, and risky paths.
|
||||
- Docs-only PRs can skip the embedded matrix without leaving the required gate
|
||||
pending, because the aggregate job still runs.
|
||||
|
||||
### Legacy Embedded Tier Retirement (D2 Step 1)
|
||||
|
||||
Since D2 step 1, `pr-risk.yml`'s legacy embedded test jobs
|
||||
(`test-embedded-storage` x5, `test-embedded-conformance` x2,
|
||||
`test-embedded-cmd` x20: 27 jobs) do not run on same-repo PRs. There,
|
||||
`pr.yml`'s gated Bazel lane `Bazel / embedded Dolt tier` (`bazel.yml`'s
|
||||
`bazel-embedded`, `--config=embedded`, the same tests and shard manifests)
|
||||
is the tier's only pre-merge run, and `CI Gate / Required` requires it to
|
||||
have run remotely and passed.
|
||||
|
||||
- Switch: the committed workflow env `BAZEL_RETIRES_LEGACY_EMBEDDED`, set to
|
||||
the same literal (`"true"` or `"false"`) in `pr.yml` and `pr-risk.yml`
|
||||
(policy-tested). It is deliberately not the `RBE_WEST_WORKERS` repo
|
||||
variable. A variable is read again by every run and re-run, and the two
|
||||
workflows are separate runs, so one could see it on and the other off;
|
||||
see "Why a committed flag" below.
|
||||
- Who: `pull_request` runs from same-repo branches (not forks) whose
|
||||
`github.actor` is not `dependabot[bot]`, while the flag is `"true"`. The
|
||||
executor secret is deliberately not part of the decision: a same-repo PR
|
||||
whose run lacks it (secret deleted or emptied) is still covered, takes
|
||||
Bazel mode `local` (no embedded lane) and so turns `CI Gate / Required`
|
||||
red, instead of quietly moving back to a legacy tier that one of its
|
||||
runs may already have skipped.
|
||||
- Everyone else keeps the legacy tier unchanged:
|
||||
- fork PRs;
|
||||
- Dependabot PRs (no Actions secrets; `github.actor`, unlike
|
||||
`github.triggering_actor`, stays `dependabot[bot]` when someone else
|
||||
re-runs them);
|
||||
- every `merge_group` run (there is no merge queue today, so this is not
|
||||
a pre-merge net);
|
||||
- every PR while the flag is `"false"`.
|
||||
|
||||
`main.yml`'s embedded jobs on push to `main` are untouched.
|
||||
- How:
|
||||
- `pr-risk.yml` and `pr.yml` each run the identical
|
||||
`bazel-embedded-coverage` job (policy-tested). The job does no checkout
|
||||
and runs no repository code. It reads the flag, the event, the fork flag
|
||||
and the actor (no secret, no variable), and outputs `covered`.
|
||||
- The three legacy test jobs add
|
||||
`needs.bazel-embedded-coverage.outputs.covered != 'true'` to their `if`.
|
||||
- PR Risk's gate accepts their skip only when `covered == true`.
|
||||
- `pr.yml`'s gate requires `BAZEL_EMBEDDED_COVERAGE` (the job's result)
|
||||
and `BAZEL_EMBEDDED_RETIRED`. The latter is red when `covered == true`
|
||||
and the Bazel call's mode is not `remote` or `BAZEL_EMBEDDED` is not
|
||||
`success`. In particular, mode `skip` (the farm switch off) and mode
|
||||
`local` (the executor secret missing) are red there, although
|
||||
`bazel-gate.sh` alone would accept them.
|
||||
- A failed, cancelled or missing decision is red in both gates.
|
||||
- `build-embedded` keeps running: its `embedded-test-binaries` artifact also
|
||||
feeds `test-proxied-cmd`, `test-server-storage` and
|
||||
`test-server-storage-full`, which are not retired in this step.
|
||||
- Why a committed flag:
|
||||
- `covered` depends on nothing a re-run can change (flag, event, fork,
|
||||
actor). So any run or re-run of PR Risk that skips the legacy tier is
|
||||
matched by `pr.yml` runs that compute the same `covered`.
|
||||
- Each such `pr.yml` run is green only if the Bazel lane ran remotely and
|
||||
passed in that run.
|
||||
- Flipping `RBE_WEST_WORKERS` either way, plus "Re-run all jobs" of
|
||||
either workflow, can no longer leave both required checks green with
|
||||
neither tier run.
|
||||
- `scripts/pr_risk_bazel_coverage_test.go` checks this by running both
|
||||
workflows' actual decision steps, `bazel.yml`'s `rbe` step and `pr.yml`'s
|
||||
actual gate step over every event, variable value, secret, fork, actor
|
||||
and flag combination, with the variable and the secret each differing
|
||||
between the two runs. It also requires the two workflows to share their
|
||||
`pull_request` triggers.
|
||||
- Cost of failing closed: while the flag is `"true"`, a same-repo PR without
|
||||
the executor secret (secret deleted or emptied, or a non-Dependabot bot
|
||||
whose runs get no secrets) is red until the secret is restored or the
|
||||
flag is committed `"false"`.
|
||||
- Revert:
|
||||
1. Commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"` in both workflows on
|
||||
`main`.
|
||||
2. Every PR needs a new push or a merge of `main` to pick it up: a
|
||||
pull_request run uses the workflow files of the PR's merge commit, and
|
||||
a re-run reuses that commit.
|
||||
3. Only then unset `RBE_WEST_WORKERS`, if remote execution should be off.
|
||||
|
||||
Reverting the D2 commits removes the decision jobs entirely.
|
||||
- Admin changes: flipping `RBE_WEST_WORKERS` or deleting the executor
|
||||
secret, followed by "Re-run all jobs" of either workflow, can no longer
|
||||
leave both required checks green with neither tier run.
|
||||
- Re-runs: "Re-run all jobs" (or "Re-run failed jobs") of one workflow
|
||||
re-evaluates variables and secrets for that workflow's run only. The
|
||||
other workflow's last result stays on the head SHA. With the committed
|
||||
flag this cannot drop the tier silently: `covered` depends on the flag
|
||||
(fixed by the merge commit), the event, the fork flag and the actor, and
|
||||
on no variable or secret. Re-running "failed jobs" keeps the earlier
|
||||
jobs' outputs, including `covered` and the `rbe` mode.
|
||||
- Enforcement scope: the beads-only ruleset requires both gates on the
|
||||
default branch (`main`) only. On PRs into `release/**` both workflows run
|
||||
and report, but merging does not wait for them, before or after this
|
||||
change. There is no merge queue and `strict` is off, so no
|
||||
pre-merge run catches a gap; only `main.yml`'s embedded jobs and
|
||||
`bazel.yml`'s push run do, after merge.
|
||||
- Lane hardening that the retirement relies on (policy-tested in
|
||||
`scripts/ci_workflow_test.go` and `scripts/pr_risk_bazel_coverage_test.go`):
|
||||
- `test:embedded --nocache_test_results`: every run executes every test,
|
||||
like the legacy `-test.count=1` jobs, so a stale or poisoned entry in
|
||||
the shared farm action cache cannot stand in for a run.
|
||||
- No retries: no `--flaky_test_attempts` or
|
||||
`--runs_per_test_detects_flakes` anywhere, and no `flaky =` other than
|
||||
a literal `False`. The lane also runs
|
||||
`bazel query 'attr(flaky, 1, tests(//...))'` before the tier and fails
|
||||
on any result, so a macro or variable cannot hide a flaky target.
|
||||
- Pinned selection:
|
||||
- The lane's `bazel test //... --config=embedded` step and the
|
||||
`--config=embedded` lines are pinned exactly.
|
||||
- No other rc line of a config the lane uses (unconfigured,
|
||||
`remote-exec`, or anything they reference) may filter, narrow,
|
||||
re-run or re-route tests.
|
||||
- `.bazelrc` imports only the two gitignored try-imports, and no other
|
||||
rc file is committed.
|
||||
- `tools/bazel/*.sh` and the whole setup-bazel action may not inject
|
||||
test selection or skips.
|
||||
- The four embedded-tagged targets' `args` and `env` are pinned.
|
||||
- `tools/bazel/check_shard_coverage.py` runs after the tier. It requires:
|
||||
- every Bazel shard to have run exactly the tests its shard script lists
|
||||
(list-only mode, minus `TestMain`, which `grep '^func Test'` lists
|
||||
but which is never a test);
|
||||
- no shard, and no conformance partition, to be all skipped.
|
||||
|
||||
A test the scripts discover from source but the Bazel binary lacks
|
||||
fails the lane instead of passing silently, and so does an injected
|
||||
`-test.short` or `BEADS_TEST_SKIP` that skips a whole shard.
|
||||
- Not changed: `conformance.yml`'s Tier 1 (`scripts/conformance.sh`) runs the
|
||||
embedded-Dolt `TestConformance` again (non-race, unsharded), duplicating
|
||||
`test-embedded-conformance` and the Bazel lane. It is not part of either
|
||||
required gate; retiring it is a separate decision.
|
||||
|
||||
### Server Dolt Storage Matrix
|
||||
|
||||
`test-server-storage-full` mirrors `test-embedded-storage`'s sharding, one
|
||||
|
|
@ -691,6 +846,10 @@ Policy for `merge_group`:
|
|||
|
||||
- `PR` and `PR Risk` must include `merge_group`.
|
||||
- `detect-ci-tier` should keep treating `merge_group` as full embedded coverage.
|
||||
- There is no merge queue today (see Current State), so these runs are not a
|
||||
pre-merge safety net for anything PRs skip; in particular they are not one
|
||||
for the retired legacy embedded tier (D2 step 1). `merge_group` keeps that
|
||||
tier only so a queue added later starts with full coverage.
|
||||
- Any risk detector added to the required topology should default to run on
|
||||
`merge_group`, because the merge group commit may combine individually safe
|
||||
PRs into a risky integration state.
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ go_test(
|
|||
"host_tools_test.go",
|
||||
"install_dolt_test.go",
|
||||
"pr_preflight_test.go",
|
||||
"pr_risk_bazel_coverage_test.go",
|
||||
"pull_dolt_image_test.go",
|
||||
"release_script_test.go",
|
||||
"test_script_test.go",
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ func TestCICapabilitySelectorWorkflowPreservesExistingAuthority(t *testing.T) {
|
|||
if err := yaml.Unmarshal(data, &workflow); err != nil {
|
||||
t.Fatalf("parse workflow: %v", err)
|
||||
}
|
||||
wantJobs := map[string]bool{"detect-ci-tier": true, "build-embedded": true, "test-embedded-storage": true, "test-embedded-conformance": true, "test-server-storage": true, "test-server-storage-full": true, "test-embedded-cmd": true, "test-proxied-cmd": true, "test-nix": true, "ci-gate": true}
|
||||
wantJobs := map[string]bool{"detect-ci-tier": true, "bazel-embedded-coverage": true, "build-embedded": true, "test-embedded-storage": true, "test-embedded-conformance": true, "test-server-storage": true, "test-server-storage-full": true, "test-embedded-cmd": true, "test-proxied-cmd": true, "test-nix": true, "ci-gate": true}
|
||||
if len(workflow.Jobs) != len(wantJobs) {
|
||||
t.Errorf("job count = %d, want %d", len(workflow.Jobs), len(wantJobs))
|
||||
}
|
||||
|
|
@ -118,18 +118,25 @@ func TestCICapabilitySelectorWorkflowPreservesExistingAuthority(t *testing.T) {
|
|||
if tier.ID != "tier" || shadow.ID != "shadow-selector" || !shadow.ContinueOnError || detector.stepIndex(t, shadow.Name) != detector.stepIndex(t, tier.Name)+1 {
|
||||
t.Errorf("shadow step is not immediately after tier: tier=%+v shadow=%+v", tier, shadow)
|
||||
}
|
||||
for _, name := range []string{"build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd"} {
|
||||
for _, name := range []string{"build-embedded", "test-server-storage", "test-server-storage-full", "test-proxied-cmd"} {
|
||||
if got := workflow.Jobs[name].If; got != "needs.detect-ci-tier.outputs.full_embedded == 'true'" {
|
||||
t.Errorf("%s if = %q", name, got)
|
||||
}
|
||||
}
|
||||
// The legacy embedded test jobs also stand down where the gated Bazel
|
||||
// lane covers them (TestPRRiskLegacyEmbeddedTierDefersToBazelLane).
|
||||
for _, name := range prRiskBazelCoveredJobs {
|
||||
if got := workflow.Jobs[name].If; got != prRiskBazelCoveredIf {
|
||||
t.Errorf("%s if = %q, want %q", name, got, prRiskBazelCoveredIf)
|
||||
}
|
||||
}
|
||||
if workflow.Jobs["test-nix"].If != "" {
|
||||
t.Errorf("test-nix if = %q, want ungated", workflow.Jobs["test-nix"].If)
|
||||
}
|
||||
gate := workflow.Jobs["ci-gate"]
|
||||
wantNeeds := []string{"detect-ci-tier", "build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd", "test-nix"}
|
||||
wantNeeds := []string{"detect-ci-tier", "bazel-embedded-coverage", "build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd", "test-nix"}
|
||||
gateStep := gate.step(t, "Evaluate CI gate")
|
||||
if gate.If != "${{ always() }}" || strings.Join(gate.Needs, ",") != strings.Join(wantNeeds, ",") || gateStep.Env["FULL_EMBEDDED"] != "${{ needs.detect-ci-tier.outputs.full_embedded }}" || gateStep.Env["CI_GATE_REQUIRED"] != "DETECT_CI_TIER BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD TEST_NIX" || !strings.Contains(gateStep.Run, "skipped_ok=\"BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD\"") {
|
||||
if gate.If != "${{ always() }}" || strings.Join(gate.Needs, ",") != strings.Join(wantNeeds, ",") || gateStep.Env["FULL_EMBEDDED"] != "${{ needs.detect-ci-tier.outputs.full_embedded }}" || gateStep.Env["CI_GATE_REQUIRED"] != "DETECT_CI_TIER BAZEL_EMBEDDED_COVERAGE BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD TEST_NIX" || !strings.Contains(gateStep.Run, "skipped_ok=\"BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD\"") {
|
||||
t.Errorf("ci-gate authority changed: needs=%v env=%v", gate.Needs, gateStep.Env)
|
||||
}
|
||||
for name, job := range workflow.Jobs {
|
||||
|
|
|
|||
|
|
@ -2048,7 +2048,8 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) {
|
|||
if !contains(gate.Needs, "bazel") {
|
||||
t.Errorf("ci-gate needs = %v, want bazel", gate.Needs)
|
||||
}
|
||||
wantBazelIDs := []string{bazelAggregateGateID}
|
||||
// Plus D2 step 1's retirement check (TestPRRiskEmbeddedDecisionMatchesBazelMode).
|
||||
wantBazelIDs := []string{bazelAggregateGateID, "BAZEL_EMBEDDED_COVERAGE", "BAZEL_EMBEDDED_RETIRED"}
|
||||
for lane, id := range bazelLaneGateIDs {
|
||||
wantBazelIDs = append(wantBazelIDs, id)
|
||||
if want := "${{ needs.bazel.outputs." + lane + " || 'skipped' }}"; evaluate.Env[id] != want {
|
||||
|
|
@ -2101,7 +2102,9 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Legacy jobs stay required: D1 adds, D2 removes.
|
||||
// Legacy jobs stay required: D1 adds, D2 removes. (D2 step 1 lets
|
||||
// pr-risk.yml's embedded test jobs skip where this lane runs remotely,
|
||||
// but they stay required ids: TestPRRiskLegacyEmbeddedTierDefersToBazelLane.)
|
||||
for id, job := range map[string]string{
|
||||
"BUILD_ARTIFACTS": "build-artifacts",
|
||||
"PR_CORE_WRAPPER": "pr-core-wrapper",
|
||||
|
|
@ -2215,8 +2218,11 @@ type bazelGateScenario struct {
|
|||
mode, enabled string // the call's rbe-mode / rbe-enabled outputs
|
||||
call string // needs.bazel.result
|
||||
outputs map[string]string // the lanes' outputs ("" = not reported)
|
||||
wantPass bool
|
||||
wantMention string // a red gate must name this id
|
||||
// pr.yml's bazel-embedded-coverage job (D2 step 1): its covered output
|
||||
// and its result ("" = success).
|
||||
covered, coverage string
|
||||
wantPass bool
|
||||
wantMention string // a red gate must name this id
|
||||
}
|
||||
|
||||
// runPRGateStep runs pr.yml's actual "Evaluate CI gate" step (its run block,
|
||||
|
|
@ -2239,6 +2245,13 @@ func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (boo
|
|||
}
|
||||
var got string
|
||||
switch {
|
||||
case m[1] == prRiskCoverageJobName && m[2] == "result":
|
||||
got = sc.coverage
|
||||
if got == "" {
|
||||
got = "success"
|
||||
}
|
||||
case m[1] == prRiskCoverageJobName && m[3] == "covered":
|
||||
got = sc.covered
|
||||
case m[1] != "bazel" && m[2] == "result":
|
||||
got = "success"
|
||||
case m[1] != "bazel":
|
||||
|
|
@ -3334,6 +3347,33 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) {
|
|||
// runs the embedded variants race like the jobs' binaries, with the jobs'
|
||||
// parallelism; each shard variant runs its job's shard script with the job's
|
||||
// shard total, and the conformance variants pass the jobs' exact selectors.
|
||||
// bazel-embedded's test step, exactly (review F4).
|
||||
const bazelEmbeddedTestRun = `set -o pipefail
|
||||
start=$(date +%s)
|
||||
rc=0
|
||||
bazel test //... --config=embedded \
|
||||
--build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \
|
||||
2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?
|
||||
echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
exit "$rc"`
|
||||
|
||||
// .bazelrc's --config=embedded, exactly and in order (review F4): no
|
||||
// --test_filter, no -test.short/-test.run/-test.skip, no retries, no
|
||||
// result caching. The conformance targets' own -test.run/-test.skip args
|
||||
// (the legacy jobs' partition) are checked against pr-risk.yml below.
|
||||
var bazelEmbeddedRCLines = []string{
|
||||
"test:embedded --@rules_go//go/config:race",
|
||||
"test:embedded --test_tag_filters=embedded",
|
||||
"test:embedded --build_tests_only",
|
||||
"test:embedded --keep_going",
|
||||
"test:embedded --test_summary=terse",
|
||||
"test:embedded --test_timeout=-1,-1,-1,1200",
|
||||
"test:embedded --test_arg=-test.parallel=4",
|
||||
"test:embedded --test_env=GO_TEST_WRAP_TESTV=1",
|
||||
"test:embedded --remote_download_regex=.*/test\\.xml$",
|
||||
"test:embedded --nocache_test_results",
|
||||
}
|
||||
|
||||
func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) {
|
||||
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName)
|
||||
test := job.step(t, "bazel test //... --config=embedded")
|
||||
|
|
@ -3344,6 +3384,13 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) {
|
|||
t.Errorf("embedded step selects remote-exec itself; setup-bazel's rc does that only when secrets are present")
|
||||
}
|
||||
assertTestStepKeepsExitStatus(t, test)
|
||||
// Review F4: the exact command line. Since D2 step 1 this lane is the
|
||||
// tier's only pre-merge run on same-repo PRs, so an extra flag (a
|
||||
// --test_filter, a --test_arg=-test.short/-test.run/-test.skip) that
|
||||
// quietly narrows it must be a reviewed edit of this test too.
|
||||
if strings.TrimSpace(test.Run) != bazelEmbeddedTestRun {
|
||||
t.Errorf("embedded step run changed; want exactly:\n%s\ngot:\n%s", bazelEmbeddedTestRun, test.Run)
|
||||
}
|
||||
// A shard with no tests assigned, or a selector that matches nothing,
|
||||
// exits 0; the job fails on any target or shard whose test.xml lists none.
|
||||
if !strings.Contains(test.Run, `--build_event_json_file="$RUNNER_TEMP/bazel-bep.json"`) {
|
||||
|
|
@ -3365,12 +3412,43 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) {
|
|||
"test:embedded --test_tag_filters=embedded",
|
||||
// The jobs pass no -parallel: GOMAXPROCS on 4-vCPU ubuntu-latest.
|
||||
"test:embedded --test_arg=-test.parallel=4",
|
||||
// The tier's only pre-merge run (D2 step 1) must execute, like the
|
||||
// legacy jobs' -test.count=1, never replay a cached result.
|
||||
"test:embedded --nocache_test_results",
|
||||
} {
|
||||
if !rc[want] {
|
||||
t.Errorf(".bazelrc lacks %q", want)
|
||||
}
|
||||
}
|
||||
// Review F4: --config=embedded is exactly these lines, in this order, and
|
||||
// no unconfigured common/build/test line (which applies to every config)
|
||||
// selects or narrows tests.
|
||||
var embeddedLines []string
|
||||
narrow := regexp.MustCompile(`test_filter|test_arg|-test\.(short|run|skip)|test_tag_filters|test_lang_filters|test_size_filters|test_timeout_filters`)
|
||||
for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
cmd, _, _ := strings.Cut(line, " ")
|
||||
if strings.HasSuffix(cmd, ":embedded") {
|
||||
embeddedLines = append(embeddedLines, line)
|
||||
}
|
||||
if (cmd == "common" || cmd == "build" || cmd == "test") && narrow.MatchString(line) {
|
||||
t.Errorf(".bazelrc %q narrows every config's tests, including --config=embedded", line)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(embeddedLines, bazelEmbeddedRCLines) {
|
||||
t.Errorf(".bazelrc --config=embedded lines changed; want exactly:\n%s\ngot:\n%s",
|
||||
strings.Join(bazelEmbeddedRCLines, "\n"), strings.Join(embeddedLines, "\n"))
|
||||
}
|
||||
if gen := readPolicyFile(t, sourceRepoRoot(t), setupBazelActionDir+"/write-bazelrc.sh"); narrow.MatchString(gen) {
|
||||
t.Errorf("setup-bazel's generated rc selects or narrows tests; it may only configure remote execution")
|
||||
}
|
||||
for line := range rc {
|
||||
// Nothing turns result caching back on for the embedded lane (a
|
||||
// later --cache_test_results wins over --nocache_test_results).
|
||||
if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") &&
|
||||
line != "test:embedded --nocache_test_results" && line != "test:docker --nocache_test_results" {
|
||||
t.Errorf(".bazelrc %q: only test:embedded and test:docker set test result caching", line)
|
||||
}
|
||||
if strings.HasPrefix(line, "test:embedded ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP")) {
|
||||
t.Errorf(".bazelrc %q: the embedded jobs run without -short and BEADS_TEST_SKIP", line)
|
||||
}
|
||||
|
|
@ -4461,3 +4539,71 @@ func TestReleaseWorkflowRestoresNoCache(t *testing.T) {
|
|||
t.Fatal("release.yml has no setup-go step; update this test")
|
||||
}
|
||||
}
|
||||
|
||||
// Review F3: the gated Bazel lanes never retry a failing test. The legacy
|
||||
// jobs they mirror (and, since D2 step 1, replace for the embedded tier) ran
|
||||
// each test once, so a retry would hide a flaky failure no other job sees.
|
||||
// Nothing may set --flaky_test_attempts (or --runs_per_test_detects_flakes,
|
||||
// which reports a failed-then-passed test as FLAKY, not FAILED): not .bazelrc,
|
||||
// a workflow, setup-bazel's generated rc, or a tools/bazel wrapper; and no
|
||||
// BUILD file or macro may mark a target flaky = True (Bazel retries those up
|
||||
// to three times by default).
|
||||
func TestBazelGatedLanesNeverRetryFlakyTests(t *testing.T) {
|
||||
if os.Getenv("TEST_SRCDIR") != "" {
|
||||
t.Skip("scripts_test's runfiles hold no other package's BUILD files")
|
||||
}
|
||||
root := sourceRepoRoot(t)
|
||||
retry := regexp.MustCompile(`flaky_test_attempts|runs_per_test_detects_flakes`)
|
||||
// Any flaky = other than a literal False/0 (a variable or macro
|
||||
// parameter could be True). bazel-embedded also asks Bazel itself
|
||||
// (TestBazelEmbeddedQueriesFlakyTargets).
|
||||
flakyAttr := regexp.MustCompile(`\bflaky\s*=\s*([^,)\s]+)`)
|
||||
checked := 0
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(root, path)
|
||||
if d.IsDir() {
|
||||
switch d.Name() {
|
||||
case ".git", "node_modules", ".beads":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if d.Type()&os.ModeSymlink != 0 {
|
||||
return nil // bazel-* convenience symlinks
|
||||
}
|
||||
base := d.Name()
|
||||
isBuild := base == "BUILD" || base == "BUILD.bazel" || strings.HasSuffix(base, ".bzl")
|
||||
isRetrySurface := rel == ".bazelrc" || strings.HasPrefix(rel, ".github"+string(filepath.Separator)) ||
|
||||
strings.HasPrefix(rel, filepath.Join("tools", "bazel")+string(filepath.Separator))
|
||||
if !isBuild && !isRetrySurface {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
checked++
|
||||
for i, line := range strings.Split(string(data), "\n") {
|
||||
code := line
|
||||
if j := strings.Index(code, "#"); j >= 0 && (isBuild || rel == ".bazelrc") {
|
||||
code = code[:j]
|
||||
}
|
||||
if retry.MatchString(code) {
|
||||
t.Errorf("%s:%d retries failing tests (%q); the gated lanes run each test once", rel, i+1, strings.TrimSpace(line))
|
||||
}
|
||||
if m := flakyAttr.FindStringSubmatch(code); isBuild && m != nil && m[1] != "False" && m[1] != "0" {
|
||||
t.Errorf("%s:%d marks a target flaky (%q); Bazel would retry it", rel, i+1, strings.TrimSpace(line))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if checked < 10 {
|
||||
t.Fatalf("checked only %d files; is the repository root right?", checked)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
1169
scripts/pr_risk_bazel_coverage_test.go
Normal file
1169
scripts/pr_risk_bazel_coverage_test.go
Normal file
File diff suppressed because it is too large
Load diff
164
tools/bazel/check_shard_coverage.py
Executable file
164
tools/bazel/check_shard_coverage.py
Executable file
|
|
@ -0,0 +1,164 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Fail unless every Bazel shard ran exactly the tests its shard script lists.
|
||||
|
||||
Usage: check_shard_coverage.py --bep <build_event_json_file> [--testlogs DIR]
|
||||
--suite LABEL SCRIPT SHARDS [--suite ...]
|
||||
[--whole LABEL ...]
|
||||
|
||||
The manifest-sharded targets (//cmd/bd:bd_embedded_test runs
|
||||
.github/scripts/embedded-test-shard.sh, ...) discover their tests from
|
||||
source with grep, but the Bazel test binary holds only the files in the
|
||||
go_test's srcs. A discovered test the binary lacks matches nothing in the
|
||||
shard's -test.run selector and passes silently, and check_testcases.py only
|
||||
rejects a shard with no tests at all.
|
||||
|
||||
For each --suite this runs SCRIPT k SHARDS with BEADS_TEST_SHARD_LIST_ONLY=1
|
||||
(from the repository root, like the CI jobs) for k = 1..SHARDS, and compares
|
||||
the names it lists with the top-level <testcase> names in the test.xml of
|
||||
Bazel shard k of LABEL in this invocation's BEP. Any test listed but absent
|
||||
(or present but not listed), a shard count other than SHARDS, a missing
|
||||
test.xml or a failing script is an error. So is a shard (or any test.xml of
|
||||
a --whole LABEL, an unsharded target such as the conformance partitions)
|
||||
whose top-level tests are all skipped: an injected -test.short,
|
||||
BEADS_TEST_SKIP or a lost BEADS_TEST_EMBEDDED_DOLT=1 turns the tier into
|
||||
t.Skip calls, which still list every test. It needs test.xml locally and
|
||||
-test.v in it, as --config=embedded sets.
|
||||
|
||||
Exit status: 0 if every shard matches, 1 otherwise.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from equivalence import read_bep, testlog_xmls # noqa: E402
|
||||
|
||||
LISTED = re.compile(r"^ (Test[A-Za-z0-9_]*)$")
|
||||
|
||||
# Names the shard scripts' `grep '^func Test'` discovery lists that go test
|
||||
# never runs as a test: TestMain(m *testing.M) is the package's test entry
|
||||
# point (internal/storage/embeddeddolt/test_fixture_test.go), never a
|
||||
# <testcase>. The legacy jobs' -test.run selector harmlessly matches nothing
|
||||
# for it. Filtered here rather than in the shard scripts so the legacy jobs'
|
||||
# selection stays byte-for-byte what it was. scripts/
|
||||
# pr_risk_bazel_coverage_test.go runs both real scripts in list-only mode and
|
||||
# requires every other listed name to be a `func Name(t *testing.T)`.
|
||||
NOT_TESTS = frozenset({"TestMain"})
|
||||
|
||||
|
||||
def listed_tests(script, shard, shards):
|
||||
"""Return the set of tests SCRIPT assigns to shard SHARD of SHARDS."""
|
||||
env = dict(os.environ, BEADS_TEST_SHARD_LIST_ONLY="1")
|
||||
out = subprocess.run(
|
||||
["bash", script, str(shard), str(shards)],
|
||||
env=env, check=True, capture_output=True, text=True,
|
||||
).stdout
|
||||
return {m.group(1) for m in map(LISTED.match, out.splitlines()) if m} - NOT_TESTS
|
||||
|
||||
|
||||
def ran_tests(xml_path):
|
||||
"""Return (set of top-level Go tests, set of those skipped) in a test.xml."""
|
||||
names, skipped = set(), set()
|
||||
for tc in ET.parse(xml_path).getroot().iter("testcase"):
|
||||
name = tc.get("name", "")
|
||||
if name.startswith("Test") and "/" not in name:
|
||||
names.add(name)
|
||||
if tc.find("skipped") is not None:
|
||||
skipped.add(name)
|
||||
return names, skipped
|
||||
|
||||
|
||||
def all_skipped_problem(where, got, skipped):
|
||||
if got and got == skipped:
|
||||
return f"{where}: every top-level test ({len(got)}) was skipped"
|
||||
return None
|
||||
|
||||
|
||||
def check(tested, testlogs, suites, lister=listed_tests, whole=()):
|
||||
"""Return ([summary lines], [problems]) for suites of (label, script, shards)
|
||||
and whole (unsharded or sharded) labels that must not be all-skipped."""
|
||||
lines, problems = [], []
|
||||
for label in whole:
|
||||
if label not in tested:
|
||||
problems.append(f"{label}: --whole, but the BEP has no result for it")
|
||||
continue
|
||||
for xml_path in testlog_xmls(testlogs, label, tested[label]):
|
||||
rel = os.path.relpath(xml_path, testlogs)
|
||||
if not os.path.exists(xml_path):
|
||||
problems.append(f"{label}: {rel} missing (was test.xml downloaded?)")
|
||||
continue
|
||||
try:
|
||||
got, skipped = ran_tests(xml_path)
|
||||
except ET.ParseError as e:
|
||||
problems.append(f"{label}: cannot parse {rel}: {e}")
|
||||
continue
|
||||
if not got:
|
||||
problems.append(f"{label}: {rel} lists no tests")
|
||||
p = all_skipped_problem(f"{label}: {rel}", got, skipped)
|
||||
if p:
|
||||
problems.append(p)
|
||||
lines.append(f"{label}: not all skipped")
|
||||
for label, script, shards in suites:
|
||||
if tested.get(label) != shards:
|
||||
problems.append(f"{label}: the BEP has {tested.get(label, 0)} shard(s), want {shards} ({script})")
|
||||
continue
|
||||
total = 0
|
||||
for k, xml_path in enumerate(testlog_xmls(testlogs, label, shards), start=1):
|
||||
try:
|
||||
want = lister(script, k, shards)
|
||||
except (OSError, subprocess.CalledProcessError) as e:
|
||||
problems.append(f"{label}: {script} {k} {shards} failed: {e}")
|
||||
continue
|
||||
rel = os.path.relpath(xml_path, testlogs)
|
||||
if not os.path.exists(xml_path):
|
||||
problems.append(f"{label}: {rel} missing (was test.xml downloaded?)")
|
||||
continue
|
||||
try:
|
||||
got, skipped = ran_tests(xml_path)
|
||||
except ET.ParseError as e:
|
||||
problems.append(f"{label}: cannot parse {rel}: {e}")
|
||||
continue
|
||||
total += len(want)
|
||||
p = all_skipped_problem(f"{label} shard {k}/{shards}", got, skipped)
|
||||
if p:
|
||||
problems.append(p)
|
||||
for name in sorted(want - got):
|
||||
problems.append(f"{label} shard {k}/{shards}: {name} is listed by {script} but did not run "
|
||||
f"(not in the target's srcs?)")
|
||||
for name in sorted(got - want):
|
||||
problems.append(f"{label} shard {k}/{shards}: {name} ran but {script} does not list it there")
|
||||
lines.append(f"{label}: {shards} shards, {total} listed tests")
|
||||
return lines, problems
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run")
|
||||
ap.add_argument("--testlogs", default=None, help="default: from the BEP, else ./bazel-testlogs")
|
||||
ap.add_argument("--suite", nargs=3, action="append", required=True, metavar=("LABEL", "SCRIPT", "SHARDS"),
|
||||
help="a manifest-sharded target, its shard script and shard count (repeatable)")
|
||||
ap.add_argument("--whole", action="append", default=[], metavar="LABEL",
|
||||
help="a target none of whose test.xml may be all skipped (repeatable)")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
suites = []
|
||||
for label, script, shards in args.suite:
|
||||
if not shards.isdigit() or int(shards) < 1:
|
||||
ap.error(f"--suite {label}: SHARDS must be a positive integer, got {shards!r}")
|
||||
suites.append((label, script, int(shards)))
|
||||
tested, _, bep_testlogs = read_bep(args.bep)
|
||||
testlogs = args.testlogs or bep_testlogs or "bazel-testlogs"
|
||||
lines, problems = check(tested, testlogs, suites, whole=args.whole)
|
||||
for line in lines:
|
||||
print(line)
|
||||
for p in problems:
|
||||
print(f"FAIL: {p}", file=sys.stderr)
|
||||
return 1 if problems else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue