diff --git a/.bazelrc b/.bazelrc index 63d000368..61e7e0207 100644 --- a/.bazelrc +++ b/.bazelrc @@ -161,6 +161,32 @@ test:doltserver-proxied --remote_download_regex=.*/test\.(log|xml)$ # visible "Re-run failed jobs" retries it. test:doltserver-proxied --experimental_remote_cache_eviction_retries=0 +# dolt-server + proxied union (item 5, Phase 1 shadow): bazel.yml's advisory +# bazel-dolt-race job runs both tiers in one invocation, beside their own +# lanes, to measure one client per group of lanes before any cutover. Exactly +# the union of --config=doltserver's and --config=doltserver-proxied's lines, +# with one tag filter naming both tiers +# (scripts/bazel_dolt_race_shadow_test.go). Every other flag is the same in +# both members or outside any action key for the other: the race build, +# -test.parallel=4 and GO_TEST_WRAP_TESTV=1 are common; the eternal timeout +# applies to no dolt-server target; the eviction retry count is what +# doltserver already gets from --config=sole-run in mode remote, the only mode +# the shadow runs in; test_summary, local_test_jobs and remote_download_regex +# are not in any action key. So each member target's test action is +# unchanged, and the two lanes' cached results carry over. The member configs +# stay for make, developers and the member lanes. +test:dolt-race --@rules_go//go/config:race +test:dolt-race --test_tag_filters=dolt-server,dolt-server-proxied +test:dolt-race --build_tests_only +test:dolt-race --keep_going +test:dolt-race --test_summary=terse +test:dolt-race --test_timeout=-1,-1,-1,1200 +test:dolt-race --test_env=GO_TEST_WRAP_TESTV=1 +test:dolt-race --test_arg=-test.parallel=4 +test:dolt-race --local_test_jobs=4 +test:dolt-race --remote_download_regex=.*/test\.(log|xml)$ +test:dolt-race --experimental_remote_cache_eviction_retries=0 + # Server-Dolt storage tier: PR Risk's "Test (Server Dolt Conformance)" and # "Test (Server Dolt Full Suite N/16)" jobs, as # //internal/storage/dolt:dolt_server_{conformance,full}_test diff --git a/.github/scripts/rrc-lane-commands.txt b/.github/scripts/rrc-lane-commands.txt index 08bd7eb5e..8821ad271 100644 --- a/.github/scripts/rrc-lane-commands.txt +++ b/.github/scripts/rrc-lane-commands.txt @@ -9,6 +9,7 @@ test //... --config=embedded test //... --config=integration test //... --config=doltserver test //... --config=doltserver-proxied +test //... --config=dolt-race test //... --config=doltserver-integration test //... --config=doltserver-cmd build --config=pure //cmd/bd:bd //cmd/bd:bd_test //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index ce9bf223b..e78832963 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -46,7 +46,11 @@ # only what the PR changed is built and tested on the runner, and it is # their only PR-time integration run (no legacy job runs it on PRs). Mode # local (rbe=off) still skips it. Direct runs (push to main, -# workflow_dispatch, nightly.yml's call) gate nothing. +# workflow_dispatch, nightly.yml's call) gate nothing. bazel-dolt-race (the +# dolt-server + proxied union shadow, mode remote on PRs and merge groups +# only) cannot fail the gate either: no output, job-level continue-on-error. +# The gate still waits for it, as for every job of the call (see its comment +# and engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "dolt-race Shadow"). # # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to @@ -1983,6 +1987,173 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # Item 5, Phase 1 (scripts/bazel_dolt_race_shadow_test.go): the advisory + # shadow of a union lane. One `bazel test --config=dolt-race` runs the + # dolt-server and proxied-server tiers together (.bazelrc: exactly the + # union of --config=doltserver's and --config=doltserver-proxied's lines, + # one tag filter for both), beside bazel-doltserver and bazel-proxied, so + # one client per group of lanes can be compared with the two it would + # replace (end time, runner minutes, test counts and results) before any + # cutover. Each member target's test action key is unchanged, so the shadow + # and the member lanes share cached results; and rbe-west deduplicates + # identical in-flight executions (confirmed by the farm maintainer: + # NativeLink 1.7.1 subscribes a second identical Execute of a cacheable + # action to the operation already in flight), so the shadow adds no + # executions. + # Advisory: it cannot fail the gate. Job-level continue-on-error (a failed + # job otherwise fails the whole call, whose result pr.yml's ci-gate + # requires as BAZEL), no workflow_call output, no job outputs, no job in + # this file needs it, and no gate script or caller names it. The gate does + # wait for it: ci-gate needs pr.yml's bazel call, which completes only when + # every job in it has finished (success, failure or timeout), so a slow + # shadow delays the gate and the merge queue. continue-on-error does not + # cover a job GitHub cancels at its own timeout-minutes, which could turn + # the call red: every step has a timeout, and the job's exceeds their sum + # (TestBazelDoltRaceShadowIsAdvisory). Mode remote only, on + # same-repo pull requests and merge groups (mode remote on a pull_request + # is never a fork): no fork mode, no cache or local mode, and not on push, + # nightly, dispatch or bazel-farm.yml's call. Its ci-analytics LANE is + # dolt-race. Removing it is the rollback; nothing reads it. + bazel-dolt-race: + name: dolt-race + needs: rbe + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.mode == 'remote' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} + # At least every step's timeout-minutes plus 5 (and the test step's plus + # 10): a hung step then fails the job (continue-on-error) before GitHub + # cancels it, which continue-on-error does not cover + # (TestBazelDoltRaceShadowIsAdvisory). bazel-proxied's steps have no + # timeouts but its test step's; these bound the rest. + timeout-minutes: 55 + continue-on-error: true + steps: + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 5 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + timeout-minutes: 1 + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + + - name: Set up Bazel + id: bazel + timeout-minutes: 5 + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} + + # As in bazel-proxied. + - name: No test target is marked flaky + timeout-minutes: 3 + run: | + flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" + if [ -n "$flaky" ]; then + echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" + exit 1 + fi + + # Both tiers' shards at once: 27 dolt-server + 35 proxied test actions, + # under the remote-exec config's 64 jobs. The step timeout is + # bazel-proxied's (the longer tier on rbe-west). + - name: bazel test //... --config=dolt-race + id: test + timeout-minutes: 25 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=dolt-race ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=dolt-race: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # The member lanes' checks over the union's BEP: bazel-proxied's (the + # dolt-server lane has none). check_shard_coverage ignores targets it + # does not list; check_testcases checks every target, so it covers the + # dolt-server tier too, which bazel-doltserver does not check: a + # dolt-server shard that runs no test turns this job red while both + # member lanes stay green. + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + timeout-minutes: 2 + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + + - name: Every listed test ran in its shard + if: ${{ always() && steps.test.outcome != 'skipped' }} + timeout-minutes: 2 + run: >- + python3 tools/bazel/check_shard_coverage.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 34 + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + timeout-minutes: 3 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: bazel-dolt-race-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 2 + env: + LANE: dolt-race + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-dolt-race-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # pr-risk.yml's server-Dolt storage tier, "Test (Server Dolt Conformance)" # and "Test (Server Dolt Full Suite N/16)", as --config=doltserver-integration: # the conformance and 16-shard full-suite variants of diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index ce91016b0..6b6a98be0 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -620,6 +620,39 @@ job's `if:` or rotating the key, in either of two ways: step's own kill switch, checked before any network call. Deleting the variable does not disable pre-warming; it restores the default of 1. +## dolt-race Shadow (Not Required) + +`bazel.yml`'s `bazel-dolt-race` job (check name `Bazel / dolt-race`) is an +advisory shadow, not a required check. It runs +`bazel test //... --config=dolt-race`, the union of `--config=doltserver` and +`--config=doltserver-proxied` with one tag filter for both tiers, beside the +`bazel-doltserver` and `bazel-proxied` lanes. Its purpose is to measure one +client per group of lanes against the two lanes it would replace (end time, +runner minutes, test counts and results) before any cutover. Each member +target's test action key is unchanged, so the shadow shares cached results +with the member lanes. + +- It runs in mode `remote` on `pull_request` and `merge_group` events only. + It never runs in a fork mode (`fork-ro`, `fork-rw`), in mode `cache` or + `local`, on `bazel-farm.yml`'s call, on push, nightly or dispatch. +- It cannot fail the gate. It has job-level `continue-on-error`, so its + failure cannot fail the call's aggregate result (`BAZEL`). It has no + `workflow_call` output, no `bazel.yml` job `needs` it, and `pr.yml`, + `ci-gate.sh` and `bazel-gate.sh` never name it. +- The gate still waits for it. `CI Gate` needs `pr.yml`'s `bazel` call, and + a reusable-workflow call completes only when all of its jobs have finished + (success, failure or timeout), `continue-on-error` ones included. A slow + shadow therefore delays `CI Gate` and the merge queue, up to its + timeout. `continue-on-error` does not cover a job GitHub cancels at its + own `timeout-minutes`, so every step has its own timeout and the job's + (65 minutes) stays at least 5 minutes above their sum and 10 above the + test step's (policy-tested). A hung step then fails the job, which + `continue-on-error` covers, before GitHub would cancel it. +- Its ci-analytics `LANE` is `dolt-race`. +- Policy: `scripts/bazel_dolt_race_shadow_test.go`. Removing the job, its + `.bazelrc` config and its line in `.github/scripts/rrc-lane-commands.txt` + is the rollback; nothing reads them. + ## Remote Repo Contents Cache A cold lane client re-runs its repository rules (gazelle's `go_deps`, the Go diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index b3d3294a1..b5664d7a6 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -35,6 +35,7 @@ go_test( srcs = [ "bazel_autofix_push_test.go", "bazel_cache_mode_test.go", + "bazel_dolt_race_shadow_test.go", "bazel_farm_workflow_test.go", "bazel_fork_mode_test.go", "bazel_go_repository_tools_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 8212d4efd..74a0de735 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -225,7 +225,9 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { // no longer forwards the app secrets either, so there is no // audience left for pre-warming in fork-ro or fork-rw (see // bazel.yml's comment on the job). - case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && strings.HasPrefix(mode, "fork-")): + // Nor does a shadow lane (bazelShadowLanes): mode remote on + // same-repo PRs and merge groups only. + case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && strings.HasPrefix(mode, "fork-")) && bazelShadowLanes[name] == "": t.Errorf("%s does not run in mode %s (every lane runs remotely)", name, mode) case mode == "cache" && (name == bazelIntegJobName || name == bazelCmdDoltJobName) && !runs: t.Errorf("%s does not run in mode cache", name) diff --git a/scripts/bazel_dolt_race_shadow_test.go b/scripts/bazel_dolt_race_shadow_test.go new file mode 100644 index 000000000..0b6008047 --- /dev/null +++ b/scripts/bazel_dolt_race_shadow_test.go @@ -0,0 +1,305 @@ +package scripts_test + +import ( + "cmp" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// The dolt-race shadow (item 5, Phase 1): bazel.yml's bazel-dolt-race job +// runs --config=dolt-race, the union of --config=doltserver and +// --config=doltserver-proxied, in one `bazel test` beside the two member +// lanes, so the union-job model can be measured against them before any +// cutover. It must be advisory: no workflow_call output, nothing in pr.yml's +// gates or bazel-gate.sh/ci-gate.sh, and a failure that cannot fail the +// call's aggregate result (BAZEL) that pr.yml's ci-gate requires. It cannot +// fail the gate, but the gate still waits for it: ci-gate needs the whole +// bazel call, which completes only when every job in it has finished. + +const ( + bazelDoltRaceJobName = "bazel-dolt-race" + bazelDoltRaceConfig = "dolt-race" + // Mode remote only, and only on same-repo pull requests and merge + // groups (mode remote on a pull_request event is never a fork: forks + // get fork-ro, fork-rw or cache). Not on push, nightly, dispatch or + // bazel-farm.yml's pull_request_target call. + bazelDoltRaceJobIf = "${{ needs.rbe.outputs.mode == 'remote' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }}" +) + +// bazelShadowLanes: bazel.yml jobs that run beside gated lanes only to +// measure a change to them, and report into nothing (no job or +// workflow_call output, no gate id, job-level continue-on-error), with the +// reason. TestBazelLaneIsGated and the continue-on-error checks carve them +// out by this map. +var bazelShadowLanes = map[string]string{ + bazelDoltRaceJobName: "item 5 Phase 1: the dolt-server + proxied union beside its member lanes, advisory until a cutover PR", +} + +// bazelDoltRaceJobTimeoutHeadroom: the minutes the shadow job's +// timeout-minutes must exceed its test step's (TestBazelDoltRaceShadowIsAdvisory). +const bazelDoltRaceJobTimeoutHeadroom = 10 + +// bazelDoltRaceJobTimeoutMargin: the minutes the shadow job's timeout-minutes +// must exceed the sum of all its step timeouts. +const bazelDoltRaceJobTimeoutMargin = 5 + +// bazelDoltRaceMergeQueueTimeout: the merge queue's check timeout (ruleset +// merge_queue.check_response_timeout_minutes). CI Gate waits for the shadow, +// so the shadow's job timeout must stay below it or a hung shadow drops the +// group from the queue. +const bazelDoltRaceMergeQueueTimeout = 60 + +// The union's members, and the tag filter each owns. +var bazelDoltRaceMembers = []struct{ config, tag string }{ + {"doltserver", "dolt-server"}, + {"doltserver-proxied", "dolt-server-proxied"}, +} + +// bazelRCConfigLines returns .bazelrc's option lines for --config=name +// (any command: test:name, build:name, ...), as "