diff --git a/.bazelrc b/.bazelrc index 0ac5774bf..8878100d0 100644 --- a/.bazelrc +++ b/.bazelrc @@ -178,23 +178,27 @@ test:doltserver-proxied --experimental_remote_cache_eviction_retries=0 # above. test:doltserver-proxied --norun_validations -# dolt-server + proxied union (item 5): bazel.yml's bazel-dolt-race job runs -# both tiers in one invocation in mode remote, in place of the bazel-doltserver -# and bazel-proxied lanes, which still run --config=doltserver and -# --config=doltserver-proxied in every other mode. Exactly the union of the two -# member configs' lines, with one tag filter naming both tiers -# (scripts/bazel_dolt_race_test.go). Every other flag is the same in both -# members or outside any action key for the other: the race build, -# -test.parallel=4 and GO_TEST_WRAP_TESTV=1 are common; the eternal timeout -# applies to no dolt-server target; the eviction retry count is what -# doltserver gets from --config=sole-run wherever it executes remotely; -# test_summary, local_test_jobs and remote_download_regex are not in any action -# key. So each member target's test action is unchanged (aquery: all 62 -# TestRunner actions byte-identical), and whichever arm the mode picks shares -# cached results with the other. The member configs stay for those modes, make -# and developers. +# Embedded + dolt-server + proxied union (item 5, A1b and Option B): +# bazel.yml's bazel-dolt-race job runs the three tiers in one invocation in +# mode remote, in place of the bazel-embedded, bazel-doltserver and +# bazel-proxied lanes, which still run --config=embedded, --config=doltserver +# and --config=doltserver-proxied in every other mode. Exactly the union of the +# three member configs' lines, with one tag filter naming all three tiers +# (scripts/bazel_dolt_race_test.go), plus one --jobs line below. Every other +# flag is the same in every member or outside any action key for the others: +# the race build, -test.parallel=4 and GO_TEST_WRAP_TESTV=1 are common; the +# eternal timeout is embedded's and proxied's own and applies to no +# dolt-server target; the eviction retry count is embedded's and proxied's own +# and what doltserver gets from --config=sole-run wherever it executes +# remotely; test_summary, local_test_jobs and both remote_download_regex lines +# (embedded's test.xml, the others' test.log|xml; the option is repeatable) are +# not in any action key. So each member target's test action is unchanged +# (aquery: all embedded, dolt-server and proxied TestRunner actions +# byte-identical to their member config's), and whichever arm the mode picks +# shares cached results with the other. The member configs stay for those +# modes, make and developers. test:dolt-race --@rules_go//go/config:race -test:dolt-race --test_tag_filters=dolt-server,dolt-server-proxied +test:dolt-race --test_tag_filters=embedded,dolt-server,dolt-server-proxied test:dolt-race --build_tests_only test:dolt-race --keep_going test:dolt-race --test_summary=terse @@ -202,12 +206,22 @@ test:dolt-race --test_timeout=-1,-1,-1,1200 test:dolt-race --test_env=GO_TEST_WRAP_TESTV=1 test:dolt-race --test_arg=-test.parallel=4 test:dolt-race --local_test_jobs=4 +test:dolt-race --remote_download_regex=.*/test\.xml$ test:dolt-race --remote_download_regex=.*/test\.(log|xml)$ test:dolt-race --experimental_remote_cache_eviction_retries=0 # The union carries its members' --norun_validations line -# (TestBazelDoltRaceConfigIsMemberUnion needs no change: both members and the -# union carry the same line). +# (TestBazelDoltRaceConfigIsMemberUnion: every member and the union carry the +# same line). test:dolt-race --norun_validations +# The one non-member line: the union's test actions all at once +# (TestBazelDoltRaceFitsTheJobsBudget), at the farm concurrency the three +# member lanes used together (embedded's --jobs=64 plus the dolt-server and +# proxied union's 62 actions). It overrides --config=remote-exec's --jobs=64, +# which the generated rc enables as `build --config=remote-exec`: a +# command-line --config expands at command-line priority, above every rc +# line (only a later command-line token could undo it; the test step's run +# is pinned). --jobs is not in any action key. +test:dolt-race --jobs=128 # Server-Dolt storage tier: PR Risk's "Test (Server Dolt Conformance)" and # "Test (Server Dolt Full Suite N/16)" jobs, as diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 7e0d8397a..5ff422b2e 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -47,13 +47,14 @@ # their only PR-time integration run (no legacy job runs it on PRs). Mode # local (rbe=off) still skips it. Direct runs (push to main, # workflow_dispatch, nightly.yml's call) gate nothing. In mode remote the -# dolt-server and proxied-server tiers run as one union job, bazel-dolt-race -# (`bazel test --config=dolt-race`), whose job.status is both the -# bazel-doltserver and the bazel-proxied output; in every other mode -# bazel-doltserver and bazel-proxied run as before and report their own. -# Exactly one of the two arms runs in any mode, so the gate ids -# (BAZEL_DOLTSERVER, BAZEL_PROXIED) and bazel-gate.sh's skips are unchanged -# (engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "dolt-race Union Lane"). +# embedded, dolt-server and proxied-server tiers run as one union job, +# bazel-dolt-race (`bazel test --config=dolt-race`), whose job.status is the +# bazel-embedded, bazel-doltserver and bazel-proxied output; in every other +# mode bazel-embedded, bazel-doltserver and bazel-proxied run as before and +# report their own. Exactly one of the two arms runs in any mode, so the gate +# ids (BAZEL_EMBEDDED, BAZEL_DOLTSERVER, BAZEL_PROXIED) and bazel-gate.sh's +# skips are unchanged (engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "dolt-race +# Union Lane"). # # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to @@ -343,9 +344,11 @@ on: bazel-release-cross: description: "bazel-release-cross's job.status" value: ${{ jobs.bazel-release-cross.outputs.result }} + # Mode remote: bazel-dolt-race runs the embedded tier too (item 5 + # Option B) and bazel-embedded skips; every other mode: the reverse. bazel-embedded: - description: "bazel-embedded's job.status" - value: ${{ jobs.bazel-embedded.outputs.result }} + description: "the embedded tier's job.status: bazel-dolt-race's in mode remote, bazel-embedded's otherwise" + value: ${{ jobs.bazel-dolt-race.outputs.result || jobs.bazel-embedded.outputs.result }} bazel-integration: description: "bazel-integration's job.status" value: ${{ jobs.bazel-integration.outputs.result }} @@ -1361,10 +1364,16 @@ jobs: # second invocation in bazel-test: the tier's long race shards stay off # bazel-test's critical path and budget, and it compiles nothing bazel-test # does not (same race configuration, shared through the remote cache). + # Not in mode remote: there bazel-dolt-race runs this tier together with + # the dolt-server and proxied-server tiers and reports for all three (item + # 5 Option B). In the fork modes there is no in-flight deduplication of one + # client's actions and the client does the work, so a union would cost the + # sum of the tiers instead of the max: this job runs there as before. bazel-embedded: name: embedded Dolt tier - # Remote execution only: skipped unless the rbe job chose remote, - # fork-ro or fork-rw (enabled; not in modes cache, local or skip). + # Remote execution only: skipped unless the rbe job chose fork-ro or + # fork-rw (enabled; not in modes cache, local or skip; in mode remote + # bazel-dolt-race runs the tier). # Locally the tier is 72 race test # processes (50 cmd/bd shards + 20 embeddeddolt storage shards + 2 # conformance partitions; ~24k test-seconds measured locally -- see @@ -1395,13 +1404,15 @@ jobs: # sum(durations)/shard_count the way a CPU-core-bound local sweep is -- # each shard is its own remote action and runs concurrently with every # other shard, farm capacity permitting. - # This lane is the tier's only PR run (pr-risk.yml's legacy jobs are - # retired) and pr.yml's gate requires it to have run here and passed. - # The runs-on expression is the shared one: Blacksmith in mode remote, - # GitHub-hosted in the fork modes (a client only). + # This lane, or bazel-dolt-race in mode remote, is the tier's only PR + # run (pr-risk.yml's legacy jobs are retired) and pr.yml's gate + # (BAZEL_EMBEDDED) requires one of them to have run and passed. The + # runs-on expression is the shared one (Blacksmith in mode remote, + # GitHub-hosted in the fork modes, a client only), although since item 5 + # Option B this job runs only in the fork modes. needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ needs.rbe.outputs.enabled == 'true' }} + if: ${{ needs.rbe.outputs.enabled == 'true' && needs.rbe.outputs.mode != 'remote' }} timeout-minutes: 60 outputs: result: ${{ steps.result.outputs.result }} @@ -2071,43 +2082,52 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" - # Item 5 (scripts/bazel_dolt_race_test.go): the dolt-server and + # Item 5 (scripts/bazel_dolt_race_test.go): the embedded, dolt-server and # proxied-server tiers in one `bazel test --config=dolt-race` (.bazelrc: - # exactly the union of --config=doltserver's and - # --config=doltserver-proxied's lines, one tag filter for both), mode - # remote only, in place of bazel-doltserver and bazel-proxied, which run - # in every other mode. One client waits on rbe-west for both tiers' - # shards at once (27 + 35 test actions, under the remote-exec config's 64 - # jobs), so the job ends with the slower tier and saves the other's - # runner. Each member target's test action key is unchanged (aquery- - # verified), so cached results carry over both ways, and a revert of the - # cutover re-executes nothing. Gated: its job.status is both the - # bazel-doltserver and the bazel-proxied workflow_call output (a failure - # in either tier fails both ids, which are both required). Every step has - # a timeout and the job's exceeds their sum by 5 minutes while staying - # under the merge queue's 60-minute check timeout + # exactly the union of --config=embedded's, --config=doltserver's and + # --config=doltserver-proxied's lines, one tag filter for all three, plus + # --jobs=128), mode remote only, in place of bazel-embedded, + # bazel-doltserver and bazel-proxied, which run in every other mode. The + # lane name stays dolt-race; since Option B its third tier is embedded (A1b + # folded in dolt-server and proxied, Option B embedded). One client waits + # on rbe-west for all three tiers' shards (182 embedded + 27 dolt-server + + # 35 proxied test actions at the cutover, in two waves under the union's + # own --jobs=128: the 64 + 62 the two lanes it replaces held together), so + # the job ends with the slowest tier and saves the other lane's runner. Each member + # target's test action key is unchanged (aquery-verified), so cached + # results carry over both ways, and a revert of the cutover re-executes + # nothing. Gated: its job.status is the bazel-embedded, bazel-doltserver + # and bazel-proxied workflow_call output (a failure in any tier fails all + # three ids, which are all required), on every event it runs on. Every + # step has a timeout and the job's exceeds their sum by 5 minutes while + # staying under the merge queue's 60-minute check timeout # (TestBazelDoltRaceJobTimeouts), so a hang fails a named step instead of - # dropping the merge group. The member jobs' configs and the union's must - # stay key-identical: the mode decides which runs (TestBazelDoltRaceConfigIsMemberUnion). - # Its ci-analytics LANE is dolt-race. + # running the job into its own timeout. The member jobs' configs and the + # union's must stay key-identical: the mode decides which runs + # (TestBazelDoltRaceConfigIsMemberUnion). Its ci-analytics LANE is + # dolt-race. bazel-dolt-race: name: dolt-race needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode == 'remote' }} # At least every step's timeout-minutes plus 5, and below the merge - # queue's 60-minute check timeout so this job alone cannot exhaust it - # (TestBazelDoltRaceJobTimeouts). Not a whole-run bound: the rbe job runs - # first and sibling lanes have longer timeouts. - timeout-minutes: 56 + # queue's 60-minute check timeout (TestBazelDoltRaceJobTimeouts). 59 + # leaves about a minute of that 60 for runner pickup and the rbe job this + # one needs, so a union that hangs all the way to its own timeout can + # still drop the merge group; accepted, since that run is red anyway. Not + # a whole-run bound: sibling lanes have longer timeouts. + timeout-minutes: 59 outputs: result: ${{ steps.result.outputs.result }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). # checkout v7 refuses a fork head on pull_request_target unless opted # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + # 3 minutes, not 5: the step sum + 5 must fit the job's 59 (a checkout + # takes seconds). - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - timeout-minutes: 5 + timeout-minutes: 3 with: ref: ${{ inputs.checkout-sha }} persist-credentials: false @@ -2149,16 +2169,21 @@ jobs: exit 1 fi - # Both tiers' shards at once: 27 dolt-server + 35 proxied test actions, - # under the remote-exec config's 64 jobs. No --config=sole-run: its one - # line (no retry on a remote cache eviction) is already in the union. - # Shards are expected around 55-95s on rbe-west (see bazel-proxied's - # test step), the proxied tier's 1200s cap included. The step timeout is - # >= proxied test_timeout 1200 s + about 3 min so Bazel writes TIMEOUT - # logs before GitHub kills the step. + # All three tiers' shards in one invocation: 182 embedded + 27 + # dolt-server + 35 proxied test actions at the cutover, two waves under + # the union's --jobs=128 (a shard that starts late in the second wave + # and hangs can be killed with the step before Bazel writes its TIMEOUT + # log; the job is red either way, only that log is lost). No + # --config=sole-run: its one line (no retry on a remote cache eviction) + # is already in the union. Every test is capped at 1200 s (rbe-west's + # limit, the embedded and proxied tiers' eternal timeout); the step + # allows that cap plus the build and the queue behind it: 28 minutes, + # past the slowest mode-remote embedded run measured (1252 s, a nightly + # --config=fresh run that re-executes everything) and >= 1200 s + about + # 3 minutes, so Bazel writes TIMEOUT logs before GitHub kills the step. - name: bazel test //... --config=dolt-race id: test - timeout-minutes: 23 + timeout-minutes: 28 run: | set -o pipefail start=$(date +%s) @@ -2172,11 +2197,14 @@ jobs: echo "bazel test --config=dolt-race: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" - # The member lanes' checks, once over the union's BEP: bazel-proxied's - # (the dolt-server lane has none). check_shard_coverage ignores targets - # it does not list; check_testcases checks every target, so it covers - # the dolt-server tier too, which bazel-doltserver does not check: a - # dolt-server shard that runs no test turns this job red. + # The member lanes' checks, once over the union's BEP: bazel-embedded's + # and bazel-proxied's (the dolt-server lane has none). check_testcases + # checks every target, so it covers the dolt-server tier too, which + # bazel-doltserver does not check: a dolt-server shard that runs no test + # turns this job red. check_shard_coverage ignores targets it does not + # list, and takes the union of the two members' --suite/--whole + # arguments (bazel-embedded's keep the 100-shard cmd block's tiling + # check over its two 50-shard targets). - name: Every target and shard ran tests if: ${{ always() && steps.test.outcome != 'skipped' }} timeout-minutes: 2 @@ -2189,6 +2217,12 @@ jobs: python3 tools/bazel/check_shard_coverage.py --bep "$RUNNER_TEMP/bazel-bep.json" --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 34 + --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 50@0/100 + --suite //cmd/bd:bd_embedded_part2_test .github/scripts/embedded-test-shard.sh 50@50/100 + --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 40 + --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test + --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_slow_test + --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} @@ -2203,6 +2237,24 @@ jobs: if-no-files-found: ignore retention-days: 7 + # bazel-embedded's served-timings upload, for the embedded tier this + # job runs in mode remote; the same step but for the artifact name, + # unique per run (the served shard manifests are regenerated from + # either). Telemetry: nothing in CI reads it, so an artifact 5xx or + # stall must not fail or hold the lane (scripts + # TestBazelEmbeddedServedTimingsUploadIsTelemetry). + - name: Upload served test timings + if: ${{ !cancelled() && steps.test.outcome != 'skipped' }} + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: bazel-dolt-race-served-testxml + path: bazel-testlogs/backend/http/**/test.xml + if-no-files-found: ignore + retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. - name: CI analytics summary id: ci-analytics @@ -2234,8 +2286,8 @@ jobs: retention-days: 7 if-no-files-found: ignore - # Last step: the job's result as the bazel-doltserver and - # bazel-proxied workflow_call outputs for pr.yml's ci-gate. always() + # Last step: the job's result as the bazel-embedded, bazel-doltserver + # and bazel-proxied workflow_call outputs for pr.yml's ci-gate. always() # runs it after a failure, a timeout or a cancellation too. - name: Record job result id: result diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 155b3b944..66e1d5f20 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -24,8 +24,9 @@ env: # The legacy Go test tiers this workflow used to run (embedded Dolt storage, # conformance and cmd shards; proxied-server cmd shards; server-Dolt storage # and full suite) are pr.yml's Bazel lanes on rbe-west now: `embedded Dolt -# tier`, `proxied-server tier` (`dolt-race`, with the dolt-server lane, in -# mode remote) and `server-Dolt storage tier` (bazel.yml), which pr.yml's `CI Gate / Required` requires to have run remotely and +# tier`, `proxied-server tier` (in mode remote both are `dolt-race`, with +# the dolt-server lane) and `server-Dolt storage tier` (bazel.yml), which +# pr.yml's `CI Gate / Required` requires to have run remotely and # passed on every pull_request and merge group, fork and Dependabot PRs # included (ga-96smfk.22). What remains is the Nix packaging check, which no # Bazel target can stand in for: it builds the package Nix users install. diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index d96187690..70057b8e8 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -74,14 +74,18 @@ go_test( # tools/bazel/pin_shards.py --shards 6 --min-seconds 3) onto 6 shards, and # the binary's round-robin runs the other ~28 (well under a second # together) in the 7th. The durations are the maximum over two sharded runs -# of the lane (bazel.yml's bazel-embedded-served-testxml artifact): 11-90s +# of the lane (bazel.yml's bazel-embedded-served-testxml artifact; in mode +# remote bazel-dolt-race runs the tier and uploads it as +# bazel-dolt-race-served-testxml): 11-90s # each, ~770s together. The same test's time varied 2-5x between those runs # with the executors' load, and sharded runs took up to 2x what the # single-shard runs did, so size the pins from sharded runs of the lane, # never from a single-shard or local one. -# Regenerate the manifest from PR runs' bazel-embedded-served-testxml -# artifacts (each http_served_test directory holds shard_*_of_7/test.xml; -# pass several runs, pin_shards.py takes each test's maximum): +# Regenerate the manifest from PR runs' served-testxml artifacts +# (bazel-dolt-race-served-testxml on same-repo runs, +# bazel-embedded-served-testxml on fork runs; each http_served_test +# directory holds shard_*_of_7/test.xml; pass several runs, pin_shards.py +# takes each test's maximum): # python3 tools/bazel/pin_shards.py --shards 6 --min-seconds 3 \ # --write backend/http/served_pinned_shards.txt \ # /http_served_test /http_served_test diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index fb418bf06..8a5e6edee 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -55,7 +55,7 @@ Current PR-related workflow names: outputs (`rbe-enabled` is `true` in `remote`, `fork-ro` and `fork-rw`); every lane exports its `job.status` as an output named after the job (in mode `remote`, the - `bazel-doltserver` and `bazel-proxied` outputs carry the `bazel-dolt-race` + `bazel-embedded`, `bazel-doltserver` and `bazel-proxied` outputs carry the `bazel-dolt-race` union job's status instead, see [dolt-race Union Lane](#dolt-race-union-lane); the `bazel-server-storage` and `bazel-cmd-dolt` outputs carry the `bazel-dolt-int` union job's status instead, see @@ -629,34 +629,57 @@ job's `if:` or rotating the key, in either of two ways: ## dolt-race Union Lane In mode `remote`, `bazel.yml`'s `bazel-dolt-race` job (check name -`Bazel / dolt-race`) runs the dolt-server and proxied-server tiers in one -`bazel test //... --config=dolt-race`, in place of the `bazel-doltserver` -(`dolt-server lane`) and `bazel-proxied` (`proxied-server tier`) jobs. -`--config=dolt-race` is exactly the union of `--config=doltserver` and -`--config=doltserver-proxied`, with one tag filter for both tiers, so every -member target's test action is unchanged (`bazel aquery`: the 27 + 35 = 62 -`TestRunner` actions are byte-identical under the member configs and the -union). One client waits on rbe-west for both tiers at once, so the job ends -with the slower tier and the other tier's runner is saved. +`Bazel / dolt-race`) runs the embedded, dolt-server and proxied-server tiers +in one `bazel test //... --config=dolt-race`, in place of the +`bazel-embedded` (`embedded Dolt tier`), `bazel-doltserver` +(`dolt-server lane`) and `bazel-proxied` (`proxied-server tier`) jobs +(item 5: A1b folded in dolt-server and proxied, Option B embedded; the lane +name stays `dolt-race`). `--config=dolt-race` is exactly the union of +`--config=embedded`, `--config=doltserver` and `--config=doltserver-proxied`, +with one tag filter for all three tiers, plus its own `--jobs=128`, so every +member target's test action is unchanged (`bazel aquery`: the 182 + 27 + 35 += 244 `TestRunner` actions are byte-identical under the member configs and +the union). One client waits on rbe-west for all three tiers (two waves of +`--jobs=128` at the cutover's 244 actions), so the +job ends with the slowest tier and the other lane's runner is saved. +- `--jobs=128` (the union's only non-member line, in no action key) is the + concurrency the two lanes it replaces held together (`bazel-embedded`'s 64 + plus the dolt-server + proxied union's 62 actions under its 64), so the + farm sees no higher peak. It overrides `--config=remote-exec`'s + `--jobs=64`: the generated rc enables that as `build --config=remote-exec`, + and a command-line `--config` expands at command-line priority, above + every rc line. - Mode `remote` only (same-repo pull requests, merge groups, push, nightly, dispatch and `bazel-farm.yml`'s authorized call). In the fork modes - (`fork-ro`, `fork-rw`), `cache` and `local`, `bazel-doltserver` and - `bazel-proxied` run as before: there a single client does both tiers' work - without in-flight deduplication, so a union would cost the sum of the - tiers rather than the max, on the run's critical path. -- Exactly one arm runs in any mode. The union's `job.status` is both the - `bazel-doltserver` and the `bazel-proxied` `workflow_call` output (each - reads the union's result first, then its member's), so `pr.yml`'s gate - ids (`BAZEL_DOLTSERVER`, `BAZEL_PROXIED`), `ci-gate.sh` and - `bazel-gate.sh` are unchanged: a failed or cancelled union reds both ids, - a union that reported nothing reads as `skipped`, which mode `remote` - never allows, and the per-mode skips are the members' as before. + (`fork-ro`, `fork-rw`), `cache` and `local`, `bazel-embedded`, + `bazel-doltserver` and `bazel-proxied` run as before: there a single + client does all tiers' work without in-flight deduplication, so a union + would cost the sum of the tiers rather than the max, on the run's critical + path. +- Exactly one arm runs in any mode. The union's `job.status` is the + `bazel-embedded`, `bazel-doltserver` and `bazel-proxied` `workflow_call` + output (each reads the union's result first, then its member's), so + `pr.yml`'s gate ids (`BAZEL_EMBEDDED`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED`), + `ci-gate.sh` and `bazel-gate.sh` are unchanged: a failed or cancelled union + reds all three ids, a union that reported nothing reads as `skipped`, which + mode `remote` never allows, and the per-mode skips are the members' as + before. All three tiers are plain-gated, so `job.status` is gate-equivalent + to a per-tier split, and every tier stays hard-gating on push, nightly and + dispatch too, where nothing reads the call outputs (unlike the dolt-int + union below, whose members' gating differs). - `check_testcases.py` and `check_shard_coverage.py` run once over the - union's BEP (the proxied tier's `--suite`; the dolt-server tier has no - shard-coverage check). The `bazel test` step has a 20-minute timeout, - every step has one, and the job's (55 minutes) is at least their sum plus - 5 and below the merge queue's 60-minute check timeout. + union's BEP (the union of the embedded and proxied tiers' `--suite` and + `--whole` arguments; the dolt-server tier has no shard-coverage check), and + the embedded tier's served-timings `test.xml` upload runs there under the + artifact name `bazel-dolt-race-served-testxml` (telemetry, + `continue-on-error`). The `bazel test` step has a 28-minute timeout (the + slowest mode-remote embedded run measured was 1252 s, a nightly fresh run), + every step has one, and the job's (59 minutes) is at least their sum plus + 5 and below the merge queue's 60-minute check timeout. That leaves about a + minute of the 60 for runner pickup and the `rbe` job, so a union that hangs + to its own timeout can still drop the merge group: accepted, since that run + is red anyway. - Its ci-analytics `LANE` is `dolt-race`. - Policy: `scripts/bazel_dolt_race_test.go`. Rollback is a revert of the cutover PR: the gate ids never changed and the action keys are equal both diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index d3a7177b3..3a7ed72a0 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -43,7 +43,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | | Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. Also runs the served http tiers with `BEADS_HTTP_TEST_REQUIRED=1`: `//backend/http:http_served_test`, `//internal/httpclient:httpclient_served_test` and the command corpus `//backend/http/corpus:corpus_served_test`, each sharded through `tools/bazel/go_test_pinned_shard.sh`. | | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 34 shards, each with a Dolt server. | -| Dolt server + proxied (`bazel-dolt-race`) | `bazel test //... --config=dolt-race` | Both rows above in one invocation; CI runs this instead of them in mode `remote`. | +| Embedded + Dolt server + proxied (`bazel-dolt-race`) | `bazel test //... --config=dolt-race` | The embedded, Dolt server and proxied rows in one invocation (`--jobs=128`); CI runs this instead of them in mode `remote`. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | | Server storage + cmd/bd Dolt server (`bazel-dolt-int`) | `bazel test //... --config=dolt-int` | The two rows above in one invocation; CI runs this instead of them in mode `remote`. No job-level `continue-on-error`: it reports a `server_storage_result` output (plain-gated, like `bazel-server-storage`) and a `cmd_dolt_result` output (advisory unless `BAZEL_CMD_DOLT_REQUIRED` is `true`, like `bazel-cmd-dolt`) separately, via a per-tier classify step over the BEP, since its members have different gating categories. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | @@ -201,8 +201,8 @@ remotely with `--config=remote-exec`. No Bazel target uses the `container` backend; run `go test` with docker and the pulled image to exercise it. PR Risk's heavier server tiers have configs of their own, run by bazel.yml only with remote execution, each in a job of its own (`bazel-proxied`, -which in mode `remote` is folded with `bazel-doltserver` into -`bazel-dolt-race`, and `bazel-server-storage`, which in mode `remote` is +which in mode `remote` is folded with `bazel-doltserver` and +`bazel-embedded` into `bazel-dolt-race`, and `bazel-server-storage`, which in mode `remote` is folded with `bazel-cmd-dolt` into `bazel-dolt-int`): `--config=doltserver-proxied` is the proxied-server cmd/bd tier ("Test (Proxied Dolt Cmd N/15)", diff --git a/scripts/bazel_dolt_race_test.go b/scripts/bazel_dolt_race_test.go index 1225344ec..dcd3ef0c1 100644 --- a/scripts/bazel_dolt_race_test.go +++ b/scripts/bazel_dolt_race_test.go @@ -13,15 +13,20 @@ import ( "testing" ) -// The dolt-race union lane (item 5): in mode remote, bazel.yml's -// bazel-dolt-race job runs --config=dolt-race, the union of -// --config=doltserver and --config=doltserver-proxied, in one `bazel test`, -// in place of the bazel-doltserver and bazel-proxied jobs; in every other -// mode those two run as before. Its job.status is both members' workflow_call -// output, so pr.yml's gate ids (BAZEL_DOLTSERVER, BAZEL_PROXIED), ci-gate.sh -// and bazel-gate.sh are unchanged. Outside mode remote one client would do -// both tiers' work with no in-flight deduplication, so the union would cost -// the sum of the two tiers rather than the max: the members stay there. +// The dolt-race union lane (item 5 A1b, and Option B for the embedded +// tier): in mode remote, bazel.yml's bazel-dolt-race job runs +// --config=dolt-race, the union of --config=embedded, --config=doltserver and +// --config=doltserver-proxied, in one `bazel test`, in place of the +// bazel-embedded, bazel-doltserver and bazel-proxied jobs; in every other +// mode those three run as before. Its job.status is every member's +// workflow_call output, so pr.yml's gate ids (BAZEL_EMBEDDED, +// BAZEL_DOLTSERVER, BAZEL_PROXIED), ci-gate.sh and bazel-gate.sh are +// unchanged, and every tier stays hard-gating on every event (unlike +// bazel-dolt-int's per-tier classify, whose single gate tier would leave the +// others non-gating on push, nightly and dispatch, where nothing reads the +// call outputs). Outside mode remote one client would do all three tiers' +// work with no in-flight deduplication, so the union would cost the sum of +// the tiers rather than the max: the members stay there. const ( bazelDoltRaceJobName = "bazel-dolt-race" @@ -34,6 +39,9 @@ const ( // execution only), minus mode remote. bazelDoltMemberIf = "${{ needs.rbe.outputs.mode != 'skip' && needs.rbe.outputs.mode != 'remote' }}" bazelProxiedMemberIf = "${{ needs.rbe.outputs.enabled == 'true' && needs.rbe.outputs.mode != 'remote' }}" + // bazel-embedded: remote execution only (fork-ro, fork-rw), as before + // Option B, minus mode remote. + bazelEmbeddedMemberIf = "${{ needs.rbe.outputs.enabled == 'true' && needs.rbe.outputs.mode != 'remote' }}" ) // bazelUnionLanes: a union job -> the gated member jobs it runs in place of @@ -41,7 +49,7 @@ const ( // (bazelLaneGateIDs) and its workflow_call output, which reads the union's // result first and the member's own otherwise (bazelLaneCallOutput). var bazelUnionLanes = map[string][]string{ - bazelDoltRaceJobName: {bazelDoltJobName, bazelProxiedJobName}, + bazelDoltRaceJobName: {bazelEmbedJobName, bazelDoltJobName, bazelProxiedJobName}, } // bazelUnionHasFlagGatedMember reports whether a union job runs in place of @@ -154,18 +162,34 @@ const bazelDoltRaceJobTimeoutMargin = 5 // every merge group, so its own timeout stays below it. const bazelDoltRaceMergeQueueTimeout = 60 -// bazelDoltRaceTestStepTimeout: the union's `bazel test` step timeout. Its -// shards run about 55-95 s on rbe-west, each capped at 1200 s; the step allows -// about 3 minutes past that cap so Bazel writes TIMEOUT logs before GitHub -// kills the step. -const bazelDoltRaceTestStepTimeout = 23 +// bazelDoltRaceTestStepTimeout: the union's `bazel test` step timeout. Every +// shard is capped at 1200 s; the slowest mode-remote embedded `bazel test` +// measured was 1252 s (a nightly --config=fresh run, 2026-10-07), so the step +// allows 28 minutes: past that run, and about 3 minutes past the cap so Bazel +// writes TIMEOUT logs before GitHub kills the step. Tighten other steps, not +// this one, if the job's sum stops fitting under the merge queue's 60. +const bazelDoltRaceTestStepTimeout = 28 -// The union's members, and the tag filter each owns. +// bazelDoltRaceJobTimeout: the job's timeout-minutes, the step sum + 5 and +// below the merge queue's 60: about a minute of that 60 is left for runner +// pickup and the rbe job, so a union that hangs all the way to its own +// timeout can still drop the group (accepted: that run is red anyway). +const bazelDoltRaceJobTimeout = 59 + +// The union's members, and the tag filter each owns, in the union filter's +// order. var bazelDoltRaceMembers = []struct{ config, tag string }{ + {"embedded", "embedded"}, {"doltserver", "dolt-server"}, {"doltserver-proxied", "dolt-server-proxied"}, } +// bazelDoltRaceJobs: the union's one line that is no member's +// (TestBazelDoltRaceFitsTheJobsBudget). --jobs is in no action key. +const bazelDoltRaceJobs = 128 + +var bazelDoltRaceNonMemberLines = []string{"test --jobs=" + strconv.Itoa(bazelDoltRaceJobs)} + // bazelRCConfigLines returns .bazelrc's option lines for --config=name // (any command: test:name, build:name, ...), as "