diff --git a/.bazelrc b/.bazelrc index 2beb0d5be..0ac5774bf 100644 --- a/.bazelrc +++ b/.bazelrc @@ -280,10 +280,23 @@ test:doltserver-cmd --keep_going test:doltserver-cmd --test_summary=terse test:doltserver-cmd --test_timeout=-1,-1,-1,1200 # As in --config=integration: Go's own timeout fires first and dumps every -# goroutine to test.log, which Bazel's kill at 1200s does not. -test:doltserver-cmd --test_arg=-test.timeout=19m +# goroutine to test.log, which Bazel's kill at 1200s does not. Item 5 A2: the +# -test.timeout=19m arg and the BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION env moved +# onto //cmd/bd:bd_dolt_server_test's and //tests/regression:regression_test's +# own args/env (the rules), so --config=dolt-int below does not need them: +# without the move, server-storage's 15m Go timeout would become 19m under +# the union, and the env would leak into server-storage's targets. The final +# argv and environment Bazel assembles (target args, then -test.timeout=19m, +# then this config's own -test.parallel=4 test_arg; same env map either way) +# are unchanged, so the remote action digest -- what RBE's cache keys on -- +# is unaffected and cached results should carry over. This IS a local +# actionKey change for these two targets' 46 actions: --test_env is part of +# the build configuration's own fingerprint, picked up by every target +# analyzed under that configuration, unlike a target's own env attribute, +# which only the target carries. To be confirmed in practice by the first +# merge group's tests_cached == tests_total in ci-analytics, not claimed here +# as already observed. test:doltserver-cmd --test_arg=-test.parallel=4 -test:doltserver-cmd --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 test:doltserver-cmd --test_env=GO_TEST_WRAP_TESTV=1 test:doltserver-cmd --local_test_jobs=4 test:doltserver-cmd --remote_download_regex=.*/test\.(log|xml)$ @@ -292,6 +305,44 @@ test:doltserver-cmd --remote_download_regex=.*/test\.(log|xml)$ # (bazel-integration's nogo-only step over //internal/testutil/integration). test:doltserver-cmd --norun_validations +# server-storage + cmd-dolt union (item 5 A2): bazel.yml's bazel-dolt-int job +# runs both tiers in one invocation in mode remote, in place of the +# bazel-server-storage and bazel-cmd-dolt jobs, which still run +# --config=doltserver-integration and --config=doltserver-cmd in every other +# mode (fork-ro, fork-rw, cache, local): there one client does both tiers' +# work with no in-flight deduplication, so a union would cost the sum of the +# tiers instead of the max. Exactly the union of the two member configs' +# lines above, with one tag filter naming both members' tags +# (scripts/bazel_dolt_int_test.go). Every other flag is the same in both +# members: the race build, tags=gms_pure_go,integration, keep_going, +# test_summary, the eternal 1200s timeout, GO_TEST_WRAP_TESTV=1, +# -test.parallel=4, local_test_jobs, remote_download_regex and +# norun_validations. So each member target's test action is unchanged in the +# sense that matters for caching (aquery: all 63 TestRunner actions' remote +# action digests are byte-identical, the key RBE's cache lookups use), though +# each action's local actionKey differs since these flags now reach the +# target via its own env/args rather than the member's build configuration. +# Cached results carry over both ways regardless. +# New for cmd-dolt here: --experimental_remote_cache_eviction_retries=0. +# server-storage already has it (above); cmd-dolt did not (it has no +# --config=sole-run of its own, unlike doltserver). Adding it to the union is +# stricter for cmd-dolt (no whole-invocation retry on a remote cache +# eviction) and policy-conformant with sole-run semantics; the eviction +# allowlist in pr_risk_bazel_coverage_test.go records the new line. +build:dolt-int --@rules_go//go/config:tags=gms_pure_go,integration +test:dolt-int --@rules_go//go/config:race +test:dolt-int --test_tag_filters=dolt-server-integration,dolt-server-cmd +test:dolt-int --build_tests_only +test:dolt-int --keep_going +test:dolt-int --test_summary=terse +test:dolt-int --test_timeout=-1,-1,-1,1200 +test:dolt-int --test_env=GO_TEST_WRAP_TESTV=1 +test:dolt-int --test_arg=-test.parallel=4 +test:dolt-int --local_test_jobs=4 +test:dolt-int --remote_download_regex=.*/test\.(log|xml)$ +test:dolt-int --norun_validations +test:dolt-int --experimental_remote_cache_eviction_retries=0 + # Integration lane: the `go test` main.yml's former "Main Linux integration" # jobs ran on push (this lane replaced them, and nightly.yml's retired Full # Test Suite, which ran the same `go test` over ./... in one job): every package diff --git a/.github/scripts/rrc-lane-commands.txt b/.github/scripts/rrc-lane-commands.txt index 8821ad271..331e18657 100644 --- a/.github/scripts/rrc-lane-commands.txt +++ b/.github/scripts/rrc-lane-commands.txt @@ -12,5 +12,6 @@ test //... --config=doltserver-proxied test //... --config=dolt-race test //... --config=doltserver-integration test //... --config=doltserver-cmd +test //... --config=dolt-int build --config=pure //cmd/bd:bd //cmd/bd:bd_test //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test build --config=js-wasm //internal/hooks:hooks_test diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 161a3bf74..8174010a2 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -357,12 +357,19 @@ on: bazel-proxied: description: "the proxied-server tier's job.status: bazel-dolt-race's in mode remote, bazel-proxied's otherwise" value: ${{ jobs.bazel-dolt-race.outputs.result || jobs.bazel-proxied.outputs.result }} + # Mode remote: bazel-dolt-int runs both tiers and the member jobs + # skip; every other mode: the reverse. One side is always empty + # (item 5 A2). bazel-dolt-int reports its own per-tier classification + # (classify_tier_results.py), not job.status: the job carries no + # continue-on-error, so job.status alone would read failure for both + # tiers on a cmd-dolt-only failure, unlike bazel-cmd-dolt's own + # advisory, continue-on-error nature outside mode remote. bazel-server-storage: - description: "bazel-server-storage's job.status" - value: ${{ jobs.bazel-server-storage.outputs.result }} + description: "the server-Dolt storage tier's result: bazel-dolt-int's server_storage_result in mode remote, bazel-server-storage's job.status otherwise" + value: ${{ jobs.bazel-dolt-int.outputs.server_storage_result || jobs.bazel-server-storage.outputs.result }} bazel-cmd-dolt: - description: "bazel-cmd-dolt's job.status (advisory unless pr.yml's BAZEL_CMD_DOLT_REQUIRED is \"true\")" - value: ${{ jobs.bazel-cmd-dolt.outputs.result }} + description: "the cmd/bd Dolt-server tier's result (advisory unless pr.yml's BAZEL_CMD_DOLT_REQUIRED is \"true\"): bazel-dolt-int's cmd_dolt_result in mode remote, bazel-cmd-dolt's job.status otherwise" + value: ${{ jobs.bazel-dolt-int.outputs.cmd_dolt_result || jobs.bazel-cmd-dolt.outputs.result }} package-mcp: description: "package-mcp's job.status (empty unless package-gates is 'on')" value: ${{ jobs.package-mcp.outputs.result }} @@ -2232,15 +2239,21 @@ jobs: # "off" skips it too, even though no caller passes that today (pr.yml and # bazel-farm.yml both pass "on") -- see the `integration` input's own # description. + # Not in mode remote: there bazel-dolt-int runs this tier together with + # the cmd/bd Dolt-server tier and reports for both (item 5 A2). In the + # fork modes and cache there is no in-flight deduplication of one client's + # actions, so a union would cost the sum of the two tiers instead of the + # max: this job runs there as before. bazel-server-storage: name: server-Dolt storage tier # Remote execution only, like bazel-integration: locally the tagged race - # build of //... alone is most of the timeout. + # build of //... alone is most of the timeout. Plus mode remote + # (bazel-dolt-int's). # This lane is the tier's only PR run (pr-risk.yml's legacy jobs are # retired) and pr.yml's gate requires it to have run here and passed. needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }} + if: ${{ needs.rbe.outputs.enabled == 'true' && needs.rbe.outputs.mode != 'remote' && inputs.integration != 'off' }} timeout-minutes: 30 outputs: result: ${{ steps.result.outputs.result }} @@ -2410,9 +2423,12 @@ jobs: # (TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers). bazel-cmd-dolt: name: cmd/bd Dolt-server tier + # Not in mode remote: there bazel-dolt-int runs this tier together with + # the server-Dolt storage tier (item 5 A2). See bazel-server-storage's + # comment above. needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04' }} - if: ${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }} + if: ${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' && needs.rbe.outputs.mode != 'remote' }} timeout-minutes: 85 continue-on-error: true outputs: @@ -2537,6 +2553,292 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # Item 5 A2 (scripts/bazel_dolt_int_test.go): the server-Dolt storage and + # cmd/bd Dolt-server tiers in one `bazel test --config=dolt-int` (.bazelrc: + # exactly the union of --config=doltserver-integration's and + # --config=doltserver-cmd's lines, one tag filter for both), mode remote + # only, in place of bazel-server-storage and bazel-cmd-dolt, which run in + # every other mode (there a single client does both tiers' work without + # in-flight deduplication, so a union would cost the sum of the two tiers + # rather than the max). One client waits on rbe-west for both tiers' shards + # at once (17 + 46 = 63 test actions, under the remote-exec config's 64 + # jobs), so the job ends with the slower tier and the other tier's runner + # is saved. cmd-dolt's Go-level -test.timeout=19m arg and its + # BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION env moved from --config=doltserver-cmd + # onto //cmd/bd:bd_dolt_server_test's and //tests/regression:regression_test's + # own args/env (their comments), so the union's tag filter alone decides + # which targets see them: without the move, server-storage's per-shard 15m + # Go timeout would become 19m under the union. The final argv and + # environment either way are identical (aquery-verified for all 4 affected + # targets, item 5 A2 PR), so remote action digests -- what RBE's cache keys + # on -- are unchanged and cached results should carry over; this is a local + # actionKey change for cmd-dolt's 46 actions, not a remote one (--test_env + # is part of the build configuration's own fingerprint, which every target + # under that configuration picks up, unlike a target's own env attribute), + # confirmed in practice by the first merge group's tests_cached == + # tests_total in ci-analytics, not claimed here as already observed. + # + # Unlike dolt-race, the members have different gating categories: + # bazel-server-storage is plain-gated (BAZEL_SERVER_STORAGE), bazel-cmd-dolt + # is flag-gated and advisory (BAZEL_CMD_DOLT, continue-on-error) until + # pr.yml's BAZEL_CMD_DOLT_REQUIRED is "true". This job therefore carries no + # job-level continue-on-error: on push, nightly and dispatch nothing reads + # BAZEL_SERVER_STORAGE off a workflow_call output, so job-level + # continue-on-error (job.status reads success regardless of the test step's + # own result) would let a server-storage failure end such a run green, + # where it must stay red exactly as it does on main today. Instead only the + # `bazel test` step carries continue-on-error, and an always() step + # (tools/bazel/classify_tier_results.py) reads the BEP by target label, + # computes each tier's own result, and fails the job itself iff + # server-storage's tier failed (a cmd-dolt-only test failure leaves the job + # green, as bazel-cmd-dolt's own continue-on-error already does outside + # mode remote) or bazel's exit code was neither 0 nor 3 (a build or infra + # failure is not attributable to one tier, so both are marked failed and + # the job fails regardless). Both members' workflow_call outputs read this + # job's matching per-tier output first, then their own member's; a job that + # dies before the classify step leaves both outputs empty, which mode + # remote's gate (bazel-gate.sh, ci-gate.sh) already treats as skipped and + # fails, same as an empty bazel-server-storage/bazel-cmd-dolt result would. + # Every step has a timeout and the job's exceeds their sum by 5 minutes + # while staying under the merge queue's 60-minute check timeout + # (TestBazelDoltIntJobTimeouts): 58 leaves only about 2 minutes for runner + # pickup and the rbe job inside that 60-minute budget, so a hang that runs + # to this job's own timeout can drop the merge group; accepted, because + # such a run is red anyway and the 58-minute job timeout bounds only this + # job, not the whole run (as #7514's dolt-race job comment). The member + # jobs' configs and the union's must stay key-identical: the mode decides + # which runs (TestBazelDoltIntConfigIsMemberUnion). Its ci-analytics LANE + # is dolt-int. + bazel-dolt-int: + name: dolt-int + needs: rbe + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.mode == 'remote' && inputs.integration != 'off' }} + # At least every step's timeout-minutes plus 5, and below the merge + # queue's 60-minute check timeout so this job alone cannot exhaust it + # (TestBazelDoltIntJobTimeouts). 58 leaves only about 2 minutes for + # runner pickup and the rbe job inside the merge queue's 60-minute check; + # a hang that runs to this timeout can therefore drop the group, which is + # accepted because such a run is red anyway and the 58-minute job timeout + # bounds this job, not the whole run (as #7514's dolt-race job comment). + timeout-minutes: 58 + outputs: + server_storage_result: ${{ steps.classify.outputs.server_storage_result }} + cmd_dolt_result: ${{ steps.classify.outputs.cmd_dolt_result }} + steps: + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 5 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + timeout-minutes: 1 + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + + - name: Set up Bazel + id: bazel + timeout-minutes: 5 + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_RRC_READ: 'true' # mode cache reads the repo contents cache through rbe-cache + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} + + # As in bazel-server-storage and bazel-proxied. + - name: No test target is marked flaky + timeout-minutes: 3 + run: | + flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" + if [ -n "$flaky" ]; then + echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" + exit 1 + fi + + # Both tiers' shards at once: 17 server-storage + 46 cmd-dolt test + # actions, under the remote-exec config's 64 jobs. No --config=sole-run: + # its one line (no retry on a remote cache eviction) is already in the + # union. The step timeout is >= the union's eternal-size test_timeout + # cap (1200s, shared by bd_dolt_server_test and regression_test) + about + # 3 min so Bazel writes TIMEOUT logs before GitHub kills the step, plus + # the margin server-storage's own step budgets for a cold compile (its + # own step was 25). continue-on-error: a cmd-dolt-only test failure + # must not fail this step (and so the job) outright; the classify step + # below decides that per-tier, from this step's own exit code (written + # to its "rc" output) and the BEP. + - name: bazel test //... --config=dolt-int + id: test + timeout-minutes: 25 + continue-on-error: true + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=dolt-int ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=dolt-int: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + echo "rc=$rc" >> "$GITHUB_OUTPUT" + exit "$rc" + + # check_testcases split per tier (should-fix 2, PR #7535 review): a + # single shared step with no continue-on-error would fail this job + # outright on a cmd-dolt-only zero-testcase shard (e.g. TestMain dying + # on dolt server start under BEADS_TEST_REQUIRE_DOLT_CONTAINER=1), + # where main (bazel-cmd-dolt alone) stays green, and the divergence + # would only show up under the documented rollback to + # BAZEL_CMD_DOLT_REQUIRED=false. Each step is --label-scoped to its own + # tier (so a --label typo or a label the run did not test fails + # closed, not open: check_testcases.py's own --label section) and + # carries continue-on-error so neither can fail the job by itself; the + # classify step below folds each step's own outcome into that tier's + # result with --check, so a cmd-dolt failure here reaches + # cmd_dolt_result only, same as a cmd-dolt BEP test failure does. + - name: Every target and shard ran tests (server-storage) + id: check-ss + if: ${{ always() && steps.test.outcome != 'skipped' }} + continue-on-error: true + timeout-minutes: 1 + run: >- + python3 tools/bazel/check_testcases.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --label //internal/storage/dolt:dolt_server_full_test + --label //internal/storage/dolt:dolt_server_conformance_test + + - name: Every target and shard ran tests (cmd-dolt) + id: check-cd + if: ${{ always() && steps.test.outcome != 'skipped' }} + continue-on-error: true + timeout-minutes: 1 + run: >- + python3 tools/bazel/check_testcases.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --label //cmd/bd:bd_dolt_server_test + --label //tests/regression:regression_test + + # bazel-server-storage's own check (cmd-dolt has none): every + # full-suite shard ran exactly the tests its shard script lists, and + # neither it nor the conformance target only skips. + - name: Every listed test ran in its shard + if: ${{ always() && steps.test.outcome != 'skipped' }} + timeout-minutes: 2 + run: >- + python3 tools/bazel/check_shard_coverage.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --suite //internal/storage/dolt:dolt_server_full_test .github/scripts/server-storage-test-shard.sh 16 + --whole //internal/storage/dolt:dolt_server_conformance_test + + # bazel-server-storage's own check, scoped with --label to the one + # target it has always budgeted: cmd-dolt's bd_dolt_server_test and + # regression_test shards are sharded too but no job budgeted them + # before the union, and this script has never judged them (see its + # docstring). + - name: Shard balance + if: ${{ always() && steps.test.outcome != 'skipped' }} + timeout-minutes: 2 + run: >- + python3 tools/bazel/shard_budget.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --label //internal/storage/dolt:dolt_server_full_test + + # failure() alone misses an ordinary test failure here: the `bazel + # test` step carries its own continue-on-error (must-fix 2), so a + # failing test does not make failure() true until the classify step + # (which runs after this one) exits non-zero. steps.test.outcome == + # 'failure' catches that case directly (nit, PR #7535 review). + - name: Upload test logs + if: ${{ (failure() || steps.test.outcome == 'failure') && steps.test.outcome != 'skipped' }} + timeout-minutes: 3 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: bazel-dolt-int-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 2 + env: + LANE: dolt-int + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-dolt-int-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + + # Last step: each tier's own result (not job.status: this job carries + # no continue-on-error, see the job's own comment) as the + # bazel-server-storage and bazel-cmd-dolt workflow_call outputs for + # pr.yml's ci-gate. always() runs it after a failure, a timeout or a + # cancellation too (steps.test.outcome stays "skipped" then, so the + # script's --bep file does not exist; it is only read when the test + # step actually ran). This step's own exit code decides job.status: it + # fails the job iff server-storage's tier failed, or `bazel test`'s + # exit code was neither 0 nor 3 (a build or infra failure, not + # attributable to one tier, fails both and the job). + - name: Classify tier results + id: classify + if: ${{ always() && steps.test.outcome != 'skipped' }} + timeout-minutes: 1 + env: + TEST_RC: ${{ steps.test.outputs.rc }} + CHECK_SS_OUTCOME: ${{ steps.check-ss.outcome }} + CHECK_CD_OUTCOME: ${{ steps.check-cd.outcome }} + run: >- + python3 tools/bazel/classify_tier_results.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --rc "$TEST_RC" + --tier server-storage=//internal/storage/dolt:dolt_server_full_test,//internal/storage/dolt:dolt_server_conformance_test + --tier cmd-dolt=//cmd/bd:bd_dolt_server_test,//tests/regression:regression_test + --gate-tier server-storage + --check "server-storage=$CHECK_SS_OUTCOME" + --check "cmd-dolt=$CHECK_CD_OUTCOME" + --github-output "$GITHUB_OUTPUT" + # F3: moved here from pr.yml so the gate depends on nothing but the rbe # job's decision (not the whole bazel call). Only pr.yml opts in (package-gates: "on"); # bazel-farm.yml and nightly.yml keep the default "off" and this job does diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 9234892a4..4a5e6605a 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1675,14 +1675,22 @@ sh_test( sh_test( name = "bd_dolt_server_test", size = "large", - # --config=doltserver-cmd's -test.timeout=19m fires (with a goroutine - # dump) before Bazel's 1200s action limit (eternal under the lane's - # --test_timeout), which kills without one. + # args' -test.timeout=19m fires (with a goroutine dump) before Bazel's + # 1200s action limit (eternal under the lane's --test_timeout), which + # kills without one. timeout = "eternal", srcs = ["//tools/bazel:go_test_pinned_shard.sh"], args = [ "$(rootpath :dolt_server_pinned_shards.txt)", "$(rootpath :bd_test)", + # Item 5 A2: moved from --config=doltserver-cmd's --test_arg + # (go_test_pinned_shard.sh forwards args after its own, so this + # lands last in argv, same as the config's --test_arg did). Go's own + # timeout fires first and dumps every goroutine to test.log, which + # Bazel's 1200s cap does not. The move keeps this out of + # --config=dolt-int, whose other member (server-storage) has a 15m + # Go timeout of its own. + "-test.timeout=19m", ], data = [ ":bd_for_tests", @@ -1700,6 +1708,9 @@ sh_test( "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", "BEADS_TEST_COMPACT_GC_FIXTURE": "$(rlocationpath :compact_gc_fixture)", "BEADS_TEST_PREFLIGHT_GO": "$(rlocationpath :preflight_go_fixture)", + # Item 5 A2: moved from --config=doltserver-cmd's --test_env, so + # --config=dolt-int does not set it for server-storage's targets too. + "BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION": "1", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", # TestSharedServerConcurrent (skipped everywhere else) as a bounded # smoke: 6 concurrent bd clients, 3 per database, against 2 diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 6590acea7..fb418bf06 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -56,7 +56,10 @@ Current PR-related workflow names: every lane exports its `job.status` as an output named after the job (in mode `remote`, the `bazel-doltserver` and `bazel-proxied` outputs carry the `bazel-dolt-race` - union job's status instead; see [dolt-race Union Lane](#dolt-race-union-lane)). + union job's status instead, see [dolt-race Union Lane](#dolt-race-union-lane); + the `bazel-server-storage` and `bazel-cmd-dolt` outputs carry the + `bazel-dolt-int` union job's status instead, see + [dolt-int Union Lane](#dolt-int-union-lane)). `pr.yml`'s gate requires the call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_RELEASE_CROSS`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, @@ -659,6 +662,112 @@ with the slower tier and the other tier's runner is saved. cutover PR: the gate ids never changed and the action keys are equal both ways, so cached results carry back. +## dolt-int Union Lane + +In mode `remote`, `bazel.yml`'s `bazel-dolt-int` job (check name +`Bazel / dolt-int`) runs the server-storage and cmd-dolt tiers in one +`bazel test //... --config=dolt-int`, in place of the `bazel-server-storage` +(`server-storage lane`) and `bazel-cmd-dolt` (`cmd-dolt tier`) jobs. +`--config=dolt-int` is exactly the union of `--config=doltserver-integration` +and `--config=doltserver-cmd`, with one tag filter for both tiers. The final +argv and environment Bazel assembles for all 63 `TestRunner` actions (17 +server-storage + 46 cmd-dolt) are unchanged under the union, so the remote +action digest -- what RBE's cache keys on -- is unaffected and cached results +should carry over (confirmed in practice by the first merge group's +`tests_cached == tests_total` in ci-analytics, not claimed here as already +observed). This is *not* a claim that the union is local-actionKey-identical +to the members: moving cmd-dolt's `-test.timeout` arg and +`BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION` env from `--config=doltserver-cmd` +onto the two cmd-dolt targets' own `args`/`env` (so the union's tag filter +alone decides which targets see them, and server-storage's Go timeout does +not change) moves them out of the build configuration's own fingerprint, +which every target under that configuration picks up, into each target's own +attributes, which only it carries -- changing the local actionKey for +cmd-dolt's 46 actions. One client waits on rbe-west for both tiers at once, +so the job ends with the slower tier and the other tier's runner is saved. + +Unlike dolt-race, this union's members do not share a gating category: +`bazel-server-storage` is plain-gated (`BAZEL_SERVER_STORAGE`), while +`bazel-cmd-dolt` is flag-gated and advisory (`BAZEL_CMD_DOLT`, +`BAZEL_CMD_DOLT_REQUIRED`). The union job therefore carries *no* job-level +`continue-on-error`: on push, nightly and dispatch nothing reads +`BAZEL_SERVER_STORAGE` off a `workflow_call` output, so a job-level +`continue-on-error` (which would make `job.status` read success regardless +of the `bazel test` step's own result) would let a server-storage failure +end such a run green, where it must stay red exactly as it does on main +today. Instead the `bazel test` step and the two per-tier "Every target and shard +ran tests" steps (`check-ss`, `check-cd`, below) each carry their own +`continue-on-error`, and an always() "Classify tier results" step +(`tools/bazel/classify_tier_results.py`) reads the BEP by target label, +folds each check step's outcome in via `--check NAME=OUTCOME`, computes each +tier's own result, and fails the job itself iff server-storage's tier failed +(a cmd-dolt-only test or zero-testcase failure leaves the job green, as +`bazel-cmd-dolt`'s own `continue-on-error` already does outside mode +`remote`); `bazel`'s exit code being neither 0 nor 3 (not attributable to one +tier) or a non-`PASSED` result for a label outside both `--tier` lists (the +label/tag lists having drifted from the BUILD files, rather than trust a +silent pass) both mark every tier failed and fail the job regardless. The +classify script's own flags are hardened to match: `--bep`/`--rc`/ +`--gate-tier`/`--github-output` may each be given only once, and a repeated +`--tier`/`--check` name is rejected, so an edit that accidentally widens or +un-gates a tier fails the script rather than silently changing what gates +the job (`scripts/bazel_dolt_int_test.go`'s exact pin of the classify step's +`run:` argv closes the same gap one level up). The job exposes two outputs, +`server_storage_result` and `cmd_dolt_result`, one per tier, instead of the +uniform `result` every other lane's always() step records. + +- Mode `remote` only (same-repo pull requests, merge groups, push, nightly, + dispatch and `bazel-farm.yml`'s authorized call), and only when `integration` + is not `"off"` (matching the members). In the fork modes (`fork-ro`, + `fork-rw`), `cache` and `local`, `bazel-server-storage` and `bazel-cmd-dolt` + run as before: there a single client does both tiers' work without + in-flight deduplication, so a union would cost the sum of the tiers rather + than the max, on the run's critical path. +- Exactly one arm runs in any mode. `bazel-server-storage`'s and + `bazel-cmd-dolt`'s `workflow_call` outputs each read the union's matching + per-tier output first, then their own member's, so `pr.yml`'s gate ids + (`BAZEL_SERVER_STORAGE`, `BAZEL_CMD_DOLT`), `ci-gate.sh` and + `bazel-gate.sh` are unchanged: a job that dies before the classify step + leaves both outputs empty, and `pr.yml` falls back an empty + `needs.bazel.outputs.bazel-server-storage`/`bazel-cmd-dolt` to the literal + `"skipped"` (`pr.yml`'s `BAZEL_SERVER_STORAGE`/`BAZEL_CMD_DOLT` env lines), + so `ci-gate.sh` takes its "was skipped" branch, not "is unset" (that branch + is only an env var that is empty after the `||` fallback, which these never + are) -- `report_failure "$var was skipped"`, the same failure as an empty + `bazel-server-storage`/`bazel-cmd-dolt` result today -- and the per-mode + skips are the members' as before. +- `check_testcases.py` runs twice over the union's BEP, once per tier + (`check-ss`, scoped with `--label` to server-storage's two targets; `check-cd`, + scoped to cmd-dolt's two), each with `continue-on-error: true` so a + zero-testcase shard in one tier reaches only that tier's classify result + (must-fix 2 of PR #7535's review), not the other's and not the job + directly. `--label`'s sets are policy-tied to the BUILD files' own + `dolt-server-integration`/`dolt-server-cmd` tag sets + (`TestBazelDoltIntTierLabelsMatchTheTags`), the same sets + `TestBazelDoltIntFitsTheJobsBudget` derives, so the two cannot drift apart. + `check_shard_coverage.py` still runs once over the union's BEP. The job + also runs a "Shard balance" step scoped with + `--label //internal/storage/dolt:dolt_server_full_test`, distinct from + `bazel-server-storage`'s own unscoped shard-balance step, since the union's + BEP otherwise mixes both tiers' shards under one report. The `bazel test` + step has a 25-minute timeout, every step has one (the two check steps + 1 minute each, preserving the pre-split step's combined 2-minute budget), + and the job's (58 minutes) is at least their sum plus 5 and below the merge + queue's 60-minute check timeout -- leaving only about 2 minutes for runner + pickup and the rbe job inside that 60-minute budget, so a hang that runs to + this job's own timeout can drop the merge group; accepted, because such a + run is red anyway and the 58-minute job timeout bounds only this job, not + the whole run. "Upload test logs" runs `if: failure() || steps.test.outcome + == 'failure'`, not `failure()` alone, since the `bazel test` step's own + `continue-on-error` means an ordinary test failure there does not make + `failure()` true until the classify step (which runs after it) exits + non-zero. +- Its ci-analytics `LANE` is `dolt-int`. +- Policy: `scripts/bazel_dolt_int_test.go`, and + `tools/bazel/classify_tier_results_test.py` for the classify script itself. + Rollback is a revert of the cutover PR: the gate ids never changed and the + remote action digests are equal both ways, so cached results carry back. + ## Remote Repo Contents Cache A cold lane client re-runs its repository rules (gazelle's `go_deps`, the Go @@ -1045,8 +1154,9 @@ Required` requires them to have run remotely and passed. (Variant A) the integration configuration (`--config=integration`'s `gms_pure_go,integration` tags) is no longer a three-lane exception: `bazel-integration` (`--config=integration`), `bazel-server-storage` - (`--config=doltserver-integration`) and `bazel-cmd-dolt` - (`--config=doltserver-cmd`) all pass `--norun_validations`. Instead + (`--config=doltserver-integration`), `bazel-cmd-dolt` + (`--config=doltserver-cmd`) and `bazel-dolt-int` (`--config=dolt-int`, + the union of the latter two) all pass `--norun_validations`. Instead `bazel-integration` runs a narrow extra build step, `Nogo: integration library files`, that validates only `//internal/testutil/integration` (the one non-test library package the `integration` build tag selects) diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 8e64bd0ca..d38f4e0fd 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -45,6 +45,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 34 shards, each with a Dolt server. | | Dolt server + proxied (`bazel-dolt-race`) | `bazel test //... --config=dolt-race` | Both rows above in one invocation; CI runs this instead of them in mode `remote`. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | +| Server storage + cmd/bd Dolt server (`bazel-dolt-int`) | `bazel test //... --config=dolt-int` | The two rows above in one invocation; CI runs this instead of them in mode `remote`. No job-level `continue-on-error`: it reports a `server_storage_result` output (plain-gated, like `bazel-server-storage`) and a `cmd_dolt_result` output (advisory unless `BAZEL_CMD_DOLT_REQUIRED` is `true`, like `bazel-cmd-dolt`) separately, via a per-tier classify step over the BEP, since its members have different gating categories. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | `make check` runs the testing.Short policy, `make ci-pr-lint` and @@ -201,7 +202,9 @@ backend; run `go test` with docker and the pulled image to exercise it. PR Risk's heavier server tiers have configs of their own, run by bazel.yml only with remote execution, each in a job of its own (`bazel-proxied`, which in mode `remote` is folded with `bazel-doltserver` into -`bazel-dolt-race`, and `bazel-server-storage`): `--config=doltserver-proxied` is the +`bazel-dolt-race`, and `bazel-server-storage`, which in mode `remote` is +folded with `bazel-cmd-dolt` into `bazel-dolt-int`): +`--config=doltserver-proxied` is the proxied-server cmd/bd tier ("Test (Proxied Dolt Cmd N/15)", `//cmd/bd:bd_proxied_test`), and `--config=doltserver-integration` the server-Dolt storage tier ("Test (Server Dolt Conformance)", "Test (Server diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index f8e72de2c..ecfe35782 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -35,6 +35,7 @@ go_test( srcs = [ "bazel_autofix_push_test.go", "bazel_cache_mode_test.go", + "bazel_dolt_int_test.go", "bazel_dolt_race_test.go", "bazel_farm_workflow_test.go", "bazel_fork_mode_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 9b1b3152e..3c5994fe4 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -295,9 +295,11 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { wantSkips = append(wantSkips, id) } } - // Flag-gated lanes' skips are accepted whatever the flag says. + // Flag-gated lanes' skips are accepted whatever the flag says, + // unless their union lane ran instead (bazel-dolt-int, for + // bazel-cmd-dolt, in mode remote: item 5 A2). for lane, g := range bazelFlagGatedLanes { - if !ran[lane] { + if !ran[lane] && !ran[bazelUnionOf(lane)] { wantSkips = append(wantSkips, g.id) } } diff --git a/scripts/bazel_dolt_int_test.go b/scripts/bazel_dolt_int_test.go new file mode 100644 index 000000000..fdb7ae447 --- /dev/null +++ b/scripts/bazel_dolt_int_test.go @@ -0,0 +1,697 @@ +package scripts_test + +import ( + "cmp" + "maps" + "os/exec" + "path" + "path/filepath" + "reflect" + "regexp" + "slices" + "strconv" + "strings" + "testing" +) + +// The dolt-int union lane (item 5 A2): in mode remote, bazel.yml's +// bazel-dolt-int job runs --config=dolt-int, the union of +// --config=doltserver-integration and --config=doltserver-cmd, in one +// `bazel test`, in place of the bazel-server-storage and bazel-cmd-dolt jobs; +// in every other mode those two run as before. Unlike dolt-race, the members +// have different gating categories: bazel-server-storage is plain-gated +// (BAZEL_SERVER_STORAGE), bazel-cmd-dolt is flag-gated and advisory +// (BAZEL_CMD_DOLT, continue-on-error) until pr.yml's BAZEL_CMD_DOLT_REQUIRED +// is "true". The union itself carries no job-level continue-on-error (see +// TestBazelDoltIntIsGated): only the `bazel test` step does, and an always() +// classify step (tools/bazel/classify_tier_results.py) computes each tier's +// own result from the BEP and fails the job iff server-storage's tier +// failed, so a cmd-dolt-only failure still leaves the job green, exactly as +// bazel-cmd-dolt's own continue-on-error already does outside mode remote. +const ( + bazelDoltIntJobName = "bazel-dolt-int" + bazelDoltIntConfig = "dolt-int" + // Mode remote and integration not "off": exactly where the members do + // not run (see #7491's members, which skip on integration == "off" too). + bazelDoltIntJobIf = "${{ needs.rbe.outputs.mode == 'remote' && inputs.integration != 'off' }}" +) + +func init() { + // bazelUnionLanes (declared in bazel_dolt_race_test.go): a union job -> the + // gated member jobs it runs in place of in mode remote. Each member keeps + // its gate id/flag and its workflow_call output, which reads the union's + // result first and the member's own otherwise (bazelLaneCallOutput). + bazelUnionLanes[bazelDoltIntJobName] = []string{bazelServerJobName, bazelCmdDoltJobName} +} + +// bazelDoltIntJobTimeoutMargin: as bazelDoltRaceJobTimeoutMargin. +const bazelDoltIntJobTimeoutMargin = 5 + +// bazelDoltIntTestStepTimeout: the union's `bazel test` step timeout. Both +// members share the same 1200 s Bazel test_timeout cap; the step allows about +// 3 minutes past that cap so Bazel writes TIMEOUT logs before GitHub kills the +// step, plus the margin server-storage's own step budgeted for a cold +// compile (its own step was 25). +const bazelDoltIntTestStepTimeout = 25 + +// bazelDoltIntClassifyRun: the classify step's run body, exactly (must-fix 3, +// PR #7535 review): pinned by equality, not substring, so an appended +// --gate-tier or a second --tier/--check cannot sneak past a Contains check +// (argparse and dict() would otherwise just let the last value silently +// win, un-gating or re-gating a tier). +const bazelDoltIntClassifyRun = `python3 tools/bazel/classify_tier_results.py --bep "$RUNNER_TEMP/bazel-bep.json" --rc "$TEST_RC" --tier server-storage=//internal/storage/dolt:dolt_server_full_test,//internal/storage/dolt:dolt_server_conformance_test --tier cmd-dolt=//cmd/bd:bd_dolt_server_test,//tests/regression:regression_test --gate-tier server-storage --check "server-storage=$CHECK_SS_OUTCOME" --check "cmd-dolt=$CHECK_CD_OUTCOME" --github-output "$GITHUB_OUTPUT"` + +// bazelDoltIntTestRun: the union's `bazel test` step body, exactly. Unlike +// the retired tiers' (bazelTierTestRun), it captures rc as a step output (no +// job-level continue-on-error to rely on job.status instead) for the +// classify step to read, alongside its own exit. +const bazelDoltIntTestRun = `set -o pipefail +start=$(date +%s) +rc=0 +bazel test //... --config=dolt-int ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? +echo "bazel test --config=dolt-int: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" +echo "rc=$rc" >> "$GITHUB_OUTPUT" +exit "$rc"` + +// The union's members, and the tag filter each owns. +var bazelDoltIntMembers = []struct{ config, tag string }{ + {"doltserver-integration", "dolt-server-integration"}, + {"doltserver-cmd", "dolt-server-cmd"}, +} + +// TestBazelDoltIntConfigIsMemberUnion: --config=dolt-int is exactly the set +// union of its members' lines, except that each member's --test_tag_filters +// is replaced by one filter naming both members' tags. Every flag stays a +// member's own, so each member target's test action is unchanged in the +// sense that matters for caching (item 5 A2, aquery-verified: byte-identical +// remote action digests, the key RBE's cache lookups use; each action's +// local actionKey differs, since dolt-int reaches these flags through the +// target's own env/args rather than the member's build configuration). Both +// arms run, the mode deciding which, so drift between them would split +// results by mode: a flag added to a member must be added to the union too. +func TestBazelDoltIntConfigIsMemberUnion(t *testing.T) { + rc := readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc") + const tagPrefix = "test --test_tag_filters=" + + var want []string + var tags []string + for _, m := range bazelDoltIntMembers { + lines := bazelRCConfigLines(rc, m.config) + if len(lines) == 0 { + t.Fatalf(".bazelrc has no --config=%s lines", m.config) + } + var memberTags []string + for _, l := range lines { + if v, ok := strings.CutPrefix(l, tagPrefix); ok { + memberTags = append(memberTags, v) + continue + } + if !slices.Contains(want, l) { + want = append(want, l) + } + } + if len(memberTags) != 1 || memberTags[0] != m.tag { + t.Errorf("--config=%s tag filters %v, want exactly [%s]", m.config, memberTags, m.tag) + } + tags = append(tags, m.tag) + } + + got := bazelRCConfigLines(rc, bazelDoltIntConfig) + var gotTags []string + var gotRest []string + for _, l := range got { + if v, ok := strings.CutPrefix(l, tagPrefix); ok { + gotTags = append(gotTags, v) + continue + } + gotRest = append(gotRest, l) + } + if wantTag := strings.Join(tags, ","); len(gotTags) != 1 || gotTags[0] != wantTag { + t.Errorf("--config=%s tag filters %v, want exactly [%s]", bazelDoltIntConfig, gotTags, wantTag) + } + if !sameStringSet(gotRest, want) { + t.Errorf("--config=%s is not the union of its members' lines (tag filter aside):\ngot:\n%s\nwant (any order):\n%s", + bazelDoltIntConfig, strings.Join(gotRest, "\n"), strings.Join(want, "\n")) + } + if len(gotRest) != len(want) { + t.Errorf("--config=%s repeats a line: %d lines for %d distinct member lines", bazelDoltIntConfig, len(gotRest), len(want)) + } + // No nested --config: the union is spelled out, so the policy above sees + // every flag. + for _, l := range got { + if strings.Contains(l, "--config=") { + t.Errorf("--config=%s line %q: spell the members' flags out, no nested --config", bazelDoltIntConfig, l) + } + } +} + +// TestBazelDoltIntSoleRunIsInTheUnion: the members never add --config=sole-run +// or read $BAZEL_SOLE_RUN (unlike bazel-test/bazel-pure/bazel-doltserver); +// each bakes --config=sole-run's one line (no retry on a remote cache +// eviction) directly and unconditionally into its own .bazelrc config +// instead. The union must carry that same line itself (not nested +// --config=sole-run: TestBazelDoltIntConfigIsMemberUnion forbids nesting), +// so both tiers' actions are the same whether the member or the union runs +// them. +func TestBazelDoltIntSoleRunIsInTheUnion(t *testing.T) { + rc := readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc") + sole := bazelRCConfigLines(rc, "sole-run") + if len(sole) == 0 { + t.Fatal(".bazelrc has no --config=sole-run lines") + } + union := bazelRCConfigLines(rc, bazelDoltIntConfig) + for _, l := range sole { + if !slices.Contains(union, l) { + t.Errorf("--config=sole-run line %q is not in --config=%s: the members' remote runs and the union would differ", l, bazelDoltIntConfig) + } + } + // And the union job takes no sole-run of its own: nothing to add. + test := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDoltIntJobName).step(t, "bazel test //... --config="+bazelDoltIntConfig) + if strings.Contains(test.Run, "BAZEL_SOLE_RUN") || strings.Contains(test.Run, "sole-run") { + t.Errorf("%s test step adds sole-run; --config=%s already has its lines", bazelDoltIntJobName, bazelDoltIntConfig) + } +} + +// TestBazelDoltIntIsGated pins the cutover: the union job carries no +// job-level continue-on-error (per-tier results come from an always() +// classify step instead: see the package comment above), it runs the union +// config with the members' checks once over its BEP (the new Shard balance +// step is bespoke, --label-scoped to server-storage's own target), and both +// members' workflow_call outputs read its per-tier result first. +func TestBazelDoltIntIsGated(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + job := workflow.job(t, bazelDoltIntJobName) + + if job.If != bazelDoltIntJobIf { + t.Errorf("%s if: %q, want %q", bazelDoltIntJobName, job.If, bazelDoltIntJobIf) + } + // Unlike dolt-race (both members plain-gated): bazel-cmd-dolt is + // flag-gated/advisory with its own continue-on-error everywhere else, + // but a job-level continue-on-error here would make job.status read + // success regardless of the `bazel test` step's own result, so a + // server-storage failure would end green on push/nightly/dispatch + // (nothing there reads BAZEL_SERVER_STORAGE off a workflow_call output + // to catch it), where it must stay red exactly as on main today. The + // job must therefore carry no continue-on-error of its own; the classify + // step's own exit code decides job.status per-tier instead. + if job.ContinueOnError { + t.Errorf("%s: want no job-level continue-on-error (per-tier results come from the classify step's exit code instead)", bazelDoltIntJobName) + } + if job.Name != "dolt-int" { + // ci-analytics' collector maps the check name "Bazel / dolt-int" to + // lane dolt-int. + t.Errorf("%s name = %q, want dolt-int", bazelDoltIntJobName, job.Name) + } + if !slices.Equal(job.Needs, []string{bazelRBEJobName}) { + t.Errorf("%s needs %v, want [%s]", bazelDoltIntJobName, job.Needs, bazelRBEJobName) + } + wantOutputs := map[string]string{ + "server_storage_result": "${{ steps.classify.outputs.server_storage_result }}", + "cmd_dolt_result": "${{ steps.classify.outputs.cmd_dolt_result }}", + } + if !reflect.DeepEqual(job.Outputs, wantOutputs) { + t.Errorf("%s outputs = %v, want %v (per-tier, from the classify step)", bazelDoltIntJobName, job.Outputs, wantOutputs) + } + + // The test step: the union config, as the member lanes run theirs, plus + // its own continue-on-error and rc capture (must-fix 2: per-tier results, + // not job-level continue-on-error; see the job's own comment). + test := job.step(t, "bazel test //... --config="+bazelDoltIntConfig) + if want := bazelDoltIntTestRun; strings.TrimSpace(test.Run) != want { + t.Errorf("%s test step run changed; want exactly:\n%s\ngot:\n%s", bazelDoltIntJobName, want, test.Run) + } + if v, ok := test.ContinueOnError.(bool); !ok || !v { + t.Errorf("%s test step: want continue-on-error: true (the classify step below decides the job's own result per-tier), got %v", bazelDoltIntJobName, test.ContinueOnError) + } + if test.ID != "test" { + t.Errorf("%s test step id = %q, want %q (the classify step reads steps.test.outcome/outputs.rc)", bazelDoltIntJobName, test.ID, "test") + } + classify := job.step(t, "Classify tier results") + if classify.ContinueOnError != nil && classify.ContinueOnError != false { + t.Errorf("%s classify step: want no continue-on-error (its own exit code must decide job.status), got %v", bazelDoltIntJobName, classify.ContinueOnError) + } + if classify.ID != "classify" { + t.Errorf("%s classify step id = %q, want %q (the job's outputs read steps.classify.outputs.*)", bazelDoltIntJobName, classify.ID, "classify") + } + if classify.If != "${{ always() && steps.test.outcome != 'skipped' }}" { + t.Errorf("%s classify step if = %q, want always() guarded by steps.test.outcome (a dead job before this step must leave both outputs empty)", bazelDoltIntJobName, classify.If) + } + // steps.test.outputs.rc and each per-tier check step's own outcome reach + // the script through env, not interpolated directly into the run script + // (TestBazelWorkflowForkFarmInputs forbids "${{" in any run: body; see + // "Record job result"'s JOB_STATUS for the existing pattern this + // follows). + for env, want := range map[string]string{ + "TEST_RC": "${{ steps.test.outputs.rc }}", + "CHECK_SS_OUTCOME": "${{ steps.check-ss.outcome }}", + "CHECK_CD_OUTCOME": "${{ steps.check-cd.outcome }}", + } { + if got := classify.Env[env]; got != want { + t.Errorf("%s classify step env.%s = %q, want %q", bazelDoltIntJobName, env, got, want) + } + } + // Pinned by exact equality, not substring (must-fix 3): see + // bazelDoltIntClassifyRun's own comment. + if got, want := strings.TrimSpace(classify.Run), bazelDoltIntClassifyRun; got != want { + t.Errorf("%s classify step run changed; want exactly:\n%s\ngot:\n%s", bazelDoltIntJobName, want, got) + } + if strings.Contains(classify.Run, "${{") { + t.Errorf("%s classify step run interpolates an expression directly: %q", bazelDoltIntJobName, classify.Run) + } + // server-storage's own "No test target is marked flaky" and "Every + // listed test ran in its shard" steps (cmd-dolt has neither): textually + // identical, once over the union's BEP. "Every target and shard ran + // tests" is bespoke per tier below (should-fix 2: split, continue-on- + // error, --label-scoped, and the result folded into that tier alone). + for member, steps := range map[string][]string{ + bazelServerJobName: {"No test target is marked flaky", "Every listed test ran in its shard"}, + } { + m := workflow.job(t, member) + for _, name := range steps { + want, got := m.step(t, name), job.step(t, name) + if strings.TrimSpace(got.Run) != strings.TrimSpace(want.Run) || got.If != want.If { + t.Errorf("%s step %q = (if %q) %q, want %s's (if %q) %q", bazelDoltIntJobName, name, got.If, got.Run, member, want.If, want.Run) + } + } + } + // The split check_testcases steps (should-fix 2, PR #7535 review): each + // --label-scoped to its own tier, continue-on-error so a zero-testcase + // shard in either tier cannot fail the job by itself -- the classify + // step's --check folds each step's own outcome into that tier's result + // instead (bazelDoltIntClassifyRun pins the --check args). + for id, want := range map[string]struct { + name string + labels []string + }{ + "check-ss": {"Every target and shard ran tests (server-storage)", []string{ + "//internal/storage/dolt:dolt_server_full_test", + "//internal/storage/dolt:dolt_server_conformance_test", + }}, + "check-cd": {"Every target and shard ran tests (cmd-dolt)", []string{ + "//cmd/bd:bd_dolt_server_test", + "//tests/regression:regression_test", + }}, + } { + step := job.step(t, want.name) + if step.ID != id { + t.Errorf("%s step %q id = %q, want %q (the classify step reads steps.%s.outcome)", bazelDoltIntJobName, want.name, step.ID, id, id) + } + if step.If != "${{ always() && steps.test.outcome != 'skipped' }}" { + t.Errorf("%s step %q if = %q, want always() guarded by steps.test.outcome", bazelDoltIntJobName, want.name, step.If) + } + if v, ok := step.ContinueOnError.(bool); !ok || !v { + t.Errorf("%s step %q: want continue-on-error: true (its outcome feeds the classify step's --check, not job.status directly), got %v", bazelDoltIntJobName, want.name, step.ContinueOnError) + } + if !strings.Contains(step.Run, "check_testcases.py") { + t.Errorf("%s step %q run = %q, want it to run check_testcases.py", bazelDoltIntJobName, want.name, step.Run) + } + for _, label := range want.labels { + if !strings.Contains(step.Run, "--label "+label) { + t.Errorf("%s step %q run missing --label %s:\n%s", bazelDoltIntJobName, want.name, label, step.Run) + } + } + } + // Shard balance is bespoke: server-storage's own step has no --label (it + // has only ever budgeted its own target), the union's scopes to that same + // target with --label so cmd-dolt's untested shards do not skew it (see + // the job's own comment and shard_budget.py's docstring). + storageShardBalance := workflow.job(t, bazelServerJobName).step(t, "Shard balance") + unionShardBalance := job.step(t, "Shard balance") + if strings.TrimSpace(unionShardBalance.Run) == strings.TrimSpace(storageShardBalance.Run) { + t.Errorf("%s Shard balance step is textually identical to %s's; want it scoped with --label", bazelDoltIntJobName, bazelServerJobName) + } + if !strings.Contains(unionShardBalance.Run, "--label //internal/storage/dolt:dolt_server_full_test") { + t.Errorf("%s Shard balance step = %q, want --label //internal/storage/dolt:dolt_server_full_test", bazelDoltIntJobName, unionShardBalance.Run) + } + + // Upload test logs (must-fix 4, PR #7535 review): the `bazel test` step + // carries its own continue-on-error, so failure() alone stays false on an + // ordinary test failure until the classify step (which runs after this + // one) exits non-zero -- by which point this step has already been + // evaluated. steps.test.outcome == 'failure' catches that case directly; + // no other test in this suite pins this job's own "Upload test logs" if + // (TestBazelWorkflowSecretsAndFailureSurface only inventories + // continue-on-error paths, and bazel_dolt_race_test.go/ci_workflow_test.go + // pin only the other lanes' upload-logs steps), so a future accidental + // revert of this condition to plain failure() would otherwise go + // unnoticed (mutation G, PR #7535 review round 3). + logs := job.step(t, "Upload test logs") + const wantLogsIf = "${{ (failure() || steps.test.outcome == 'failure') && steps.test.outcome != 'skipped' }}" + if logs.If != wantLogsIf { + t.Errorf("%s Upload test logs if = %q, want %q", bazelDoltIntJobName, logs.If, wantLogsIf) + } + if logs.With["name"] != "bazel-dolt-int-testlogs" { + t.Errorf("%s Upload test logs name = %q, want bazel-dolt-int-testlogs", bazelDoltIntJobName, logs.With["name"]) + } + + for _, member := range bazelUnionLanes[bazelDoltIntJobName] { + for _, step := range workflow.job(t, member).Steps { + if strings.Contains(step.Run, "shard_budget.py") && member != bazelServerJobName { + t.Errorf("%s step %q: a per-lane check the union does not run", member, step.Name) + } + if strings.Contains(step.Run, "check_shard_coverage.py") && member != bazelServerJobName { + t.Errorf("%s step %q: a per-lane check the union does not run", member, step.Name) + } + } + } + + // No job depends on it, and every member's output reads it first. + for name, other := range workflow.Jobs { + if slices.Contains(other.Needs, bazelDoltIntJobName) { + t.Errorf("job %s needs %s", name, bazelDoltIntJobName) + } + } + call := readBazelWorkflowCall(t) + for _, member := range bazelUnionLanes[bazelDoltIntJobName] { + if got, want := call.Outputs[member].Value, bazelLaneCallOutput(member); got != want { + t.Errorf("workflow_call output %s = %q, want %q", member, got, want) + } + } + for name := range call.Outputs { + if strings.Contains(name, bazelDoltIntConfig) { + t.Errorf("workflow_call output %s: the union reports through its members' outputs, not one of its own (the gate ids stay)", name) + } + } +} + +// TestBazelDoltIntJobTimeouts: the union's test step has a 25-minute timeout, +// every step has one, and the job's exceeds their sum by +// bazelDoltIntJobTimeoutMargin minutes (a hang fails a named step) while +// staying under the merge queue's check timeout. +func TestBazelDoltIntJobTimeouts(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDoltIntJobName) + test := job.step(t, "bazel test //... --config="+bazelDoltIntConfig) + if test.TimeoutMinutes != bazelDoltIntTestStepTimeout { + t.Errorf("%s test step timeout-minutes = %d, want %d", bazelDoltIntJobName, test.TimeoutMinutes, bazelDoltIntTestStepTimeout) + } + sum := 0 + for i, step := range job.Steps { + if step.TimeoutMinutes <= 0 { + t.Errorf("%s step %d (%q) has no timeout-minutes; a hang would run into the job timeout", + bazelDoltIntJobName, i, cmp.Or(step.Name, step.Uses)) + } + sum += step.TimeoutMinutes + } + if job.TimeoutMinutes < sum+bazelDoltIntJobTimeoutMargin { + t.Errorf("%s timeout-minutes %d, want at least the sum of its step timeouts (%d) + %d", + bazelDoltIntJobName, job.TimeoutMinutes, sum, bazelDoltIntJobTimeoutMargin) + } + if job.TimeoutMinutes >= bazelDoltRaceMergeQueueTimeout { + t.Errorf("%s timeout-minutes %d, want below the merge queue's %d-minute check timeout", + bazelDoltIntJobName, job.TimeoutMinutes, bazelDoltRaceMergeQueueTimeout) + } +} + +// TestBazelDoltIntArmsExactlyOnePerMode evaluates the union's and the +// members' if: for every execution mode and every event bazel.yml runs on: +// the union runs exactly in mode remote, never beside a member, and each +// member's gate id/flag is reported by exactly one job wherever that member +// ran before the union (bazel-server-storage: the remote-executing modes +// plus cache, when enabled and integration is not off; bazel-cmd-dolt: where +// bazel-integration runs) and by none elsewhere, so bazel-gate.sh's per-mode +// skips are unchanged. The workflow_call outputs, evaluated as GitHub does, +// carry the running arm's result: a failed or cancelled union reds both +// ids/flags, and a union that never reported reads as skipped, which no mode +// it runs in allows. +func TestBazelDoltIntArmsExactlyOnePerMode(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + union := workflow.job(t, bazelDoltIntJobName) + members := map[string]struct { + ifExpr string + before func(mode string) bool // where the member ran before the union + }{ + bazelServerJobName: {bazelServerStorageIf, func(m string) bool { return bazelRemoteModes[m] }}, + bazelCmdDoltJobName: {bazelCmdDoltMemberIf, func(m string) bool { return bazelRemoteModes[m] || m == "cache" }}, + } + if got := slices.Sorted(maps.Keys(members)); !slices.Equal(got, slices.Sorted(slices.Values(bazelUnionLanes[bazelDoltIntJobName]))) { + t.Fatalf("members %v, want bazelUnionLanes' %v", got, bazelUnionLanes[bazelDoltIntJobName]) + } + for name, m := range members { + if got := workflow.job(t, name).If; got != m.ifExpr { + t.Errorf("%s if = %q, want %q", name, got, m.ifExpr) + } + } + for _, mode := range bazelRBEModes { + for _, event := range []string{"pull_request", "merge_group", "pull_request_target", "push", "schedule", "workflow_dispatch"} { + for _, integration := range []string{"", "off"} { + ctx := map[string]string{ + "needs.rbe.outputs.mode": mode, + "needs.rbe.outputs.enabled": bazelModeEnabled(mode), + "github.event_name": event, + "inputs.integration": integration, + } + wantUnionRuns := mode == "remote" && integration != "off" + unionRuns := evalRRCIf(t, union.If, ctx) + if unionRuns != wantUnionRuns { + t.Errorf("mode %s, %s, integration %q: %s runs = %v, want %v", mode, event, integration, bazelDoltIntJobName, unionRuns, wantUnionRuns) + } + for name, m := range members { + runs := evalRRCIf(t, workflow.job(t, name).If, ctx) + if unionRuns && runs { + t.Errorf("mode %s, %s, integration %q: %s and %s both run", mode, event, integration, bazelDoltIntJobName, name) + } + wantBefore := integration != "off" && m.before(mode) + if got, want := unionRuns || runs, wantBefore; got != want { + t.Errorf("mode %s, %s, integration %q: %s's tier reported by a job = %v, want %v (where it ran before the union)", mode, event, integration, name, got, want) + } + for _, res := range []string{"success", "failure", "cancelled", ""} { + jobs := map[string]string{} + switch { + case unionRuns: + jobs[bazelDoltIntJobName] = res + case runs: + jobs[name] = res + } + want := res + if !unionRuns && !runs { + want = "" + } + if got := bazelCallOutputValue(t, name, jobs); got != want { + t.Errorf("mode %s, %s, integration %q, reporting job %q: output %s = %q, want %q", mode, event, integration, res, name, got, want) + } + } + } + } + } + } + // The union's runner is the lanes' ternary: Blacksmith in mode remote, + // the only mode it runs in. + if union.RunsOn != bazelLaneRunsOn { + t.Errorf("%s runs-on = %q, want %q", bazelDoltIntJobName, union.RunsOn, bazelLaneRunsOn) + } +} + +// classifyTierResultsTest: tools/bazel/classify_tier_results_test.py's own +// path, run directly (its docstring), not through a Bazel target: locally +// authored, so no SHA256 content pin is needed (unlike the vendored-from- +// gascity Python test files this repo also wraps, e.g. bazel_rrc_test.go's +// rrc_verify_test.py). +const classifyTierResultsTest = "tools/bazel/classify_tier_results_test.py" +const checkTestcasesTest = "tools/bazel/check_testcases_test.py" + +// TestClassifyTierResultsUnitTests runs classify_tier_results.py's own +// behavioural test (must-fix 2(f)): the script the "Classify tier results" +// step above calls, covering ss-fail-only, cmd-fail-only, both, build +// failure and rc=0 over a synthetic BEP/test summary. +func TestClassifyTierResultsUnitTests(t *testing.T) { + root := sourceRepoRoot(t) + python := requireHostTool(t, "python3") + cmd := exec.Command(python, filepath.Join(root, classifyTierResultsTest), "-v") + cmd.Dir = filepath.Join(root, "tools", "bazel") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%s failed: %v\n%s", classifyTierResultsTest, err, out) + } +} + +// TestCheckTestcasesUnitTests runs check_testcases.py's own behavioural test +// (should-fix 2 of PR #7535's review): the script the union's two per-tier +// "Every target and shard ran tests" steps (check-ss, check-cd) each call +// with their own --label set, covering that --label both restricts the +// check to its own tier's targets and still fails closed on a label the BEP +// has no result for, or a zero-testcase shard within its own tier. +func TestCheckTestcasesUnitTests(t *testing.T) { + root := sourceRepoRoot(t) + python := requireHostTool(t, "python3") + cmd := exec.Command(python, filepath.Join(root, checkTestcasesTest), "-v") + cmd.Dir = filepath.Join(root, "tools", "bazel") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%s failed: %v\n%s", checkTestcasesTest, err, out) + } +} + +// bazelDoltIntTestActions: the union's test actions, 17 server-storage + 46 +// cmd-dolt (aquery, item 5 A2). +const bazelDoltIntTestActions = 63 + +// TestBazelDoltIntFitsTheJobsBudget: the union runs both tiers' shards in one +// invocation under --config=remote-exec's --jobs, so all of them must fit at +// once or the slower tier's shards queue behind the other's. Every rule +// tagged with one of the union's tags, read from the BUILD files: its +// shard_count (1 when unsharded), summed. +func TestBazelDoltIntFitsTheJobsBudget(t *testing.T) { + root := bazelPolicyRoot(t) + rc := readPolicyFile(t, root, ".bazelrc") + m := regexp.MustCompile(`(?m)^build:remote-exec --jobs=(\d+)$`).FindStringSubmatch(rc) + if m == nil { + t.Fatal(".bazelrc has no build:remote-exec --jobs=N line") + } + budget, _ := strconv.Atoi(m[1]) + + var tags []string + for _, mem := range bazelDoltIntMembers { + tags = append(tags, `"`+mem.tag+`"`) + } + ruleStart := regexp.MustCompile(`(?m)^[A-Za-z_][A-Za-z0-9_]*\($`) + total, targets := 0, 0 + for _, f := range repoFiles(t, root, 300) { + if path.Base(f) != "BUILD.bazel" { + continue + } + build := readPolicyFile(t, root, f) + for _, line := range strings.Split(build, "\n") { + for _, tag := range tags { + if strings.Contains(line, tag) && !strings.HasPrefix(line, " tags = [") { + t.Errorf("%s: %q outside a one-line tags list: %q", f, tag, line) + } + } + } + for _, loc := range ruleStart.FindAllStringIndex(build, -1) { + end := strings.Index(build[loc[0]:], "\n)\n") + if end < 0 { + continue + } + rule := build[loc[0] : loc[0]+end] + tagged := false + for _, line := range strings.Split(rule, "\n") { + if strings.HasPrefix(line, " tags = [") && slices.ContainsFunc(tags, func(tag string) bool { return strings.Contains(line, tag) }) { + tagged = true + } + } + if !tagged { + continue + } + shards := 1 + if sm := shardCountPattern.FindStringSubmatch(rule); sm != nil { + shards, _ = strconv.Atoi(sm[1]) + } + total += shards + targets++ + } + } + if total != bazelDoltIntTestActions { + t.Errorf("--config=%s: %d test actions over %d targets, want %d (aquery-verified at the cutover): re-check against --jobs=%d and update", bazelDoltIntConfig, total, targets, bazelDoltIntTestActions, budget) + } + if total > budget { + t.Errorf("--config=%s: %d test actions, more than --config=remote-exec's --jobs=%d: the tiers would no longer run at once", bazelDoltIntConfig, total, budget) + } +} + +// bazelDoltIntTierNames maps each member's tag to the --tier/--check name +// bazel.yml's classify step uses for it. +var bazelDoltIntTierNames = map[string]string{ + "dolt-server-integration": "server-storage", + "dolt-server-cmd": "cmd-dolt", +} + +// bazelDoltIntTaggedLabels returns, for each of the union's tags, the exact +// sorted set of labels the BUILD files tag with it: the same rule-block walk +// TestBazelDoltIntFitsTheJobsBudget uses to sum shard counts, but keeping +// each rule's own label (//pkg:name) instead. +func bazelDoltIntTaggedLabels(t *testing.T) map[string][]string { + t.Helper() + root := bazelPolicyRoot(t) + ruleStart := regexp.MustCompile(`(?m)^[A-Za-z_][A-Za-z0-9_]*\($`) + nameLine := regexp.MustCompile(`(?m)^ name = "([^"]+)",$`) + out := map[string][]string{} + for _, f := range repoFiles(t, root, 300) { + if path.Base(f) != "BUILD.bazel" { + continue + } + build := readPolicyFile(t, root, f) + pkg := path.Dir(f) + if pkg == "." { + pkg = "" + } + for _, loc := range ruleStart.FindAllStringIndex(build, -1) { + end := strings.Index(build[loc[0]:], "\n)\n") + if end < 0 { + continue + } + rule := build[loc[0] : loc[0]+end] + nm := nameLine.FindStringSubmatch(rule) + if nm == nil { + continue + } + for _, mem := range bazelDoltIntMembers { + tagLine := `"` + mem.tag + `"` + for _, line := range strings.Split(rule, "\n") { + if strings.HasPrefix(line, " tags = [") && strings.Contains(line, tagLine) { + out[mem.tag] = append(out[mem.tag], "//"+pkg+":"+nm[1]) + break + } + } + } + } + } + for tag := range out { + slices.Sort(out[tag]) + } + return out +} + +// TestBazelDoltIntTierLabelsMatchTheTags ties the classify step's --tier +// label lists, and the per-tier check_testcases.py steps' --label lists, to +// the BUILD files' own tag sets -- the same sets TestBazelDoltIntFitsTheJobsBudget +// walks (must-fix 1, PR #7535 review): classify_tier_results.py fails every +// tier closed on a non-PASSED result for a label outside every --tier, but +// only if these lists actually name every tagged target. A target tagged +// dolt-server-integration or dolt-server-cmd without a matching update here +// (and to bazelDoltIntTestActions/bazelDoltIntClassifyRun, which this test +// also indirectly guards) would otherwise run untracked. +func TestBazelDoltIntTierLabelsMatchTheTags(t *testing.T) { + tagged := bazelDoltIntTaggedLabels(t) + workflow := readCIWorkflow(t, bazelWorkflowName) + job := workflow.job(t, bazelDoltIntJobName) + + classify := job.step(t, "Classify tier results") + tierArg := regexp.MustCompile(`--tier ([a-z-]+)=(\S+)`) + gotTiers := map[string][]string{} + for _, m := range tierArg.FindAllStringSubmatch(classify.Run, -1) { + gotTiers[m[1]] = strings.Split(m[2], ",") + } + + checkSteps := map[string]string{ + "dolt-server-integration": "Every target and shard ran tests (server-storage)", + "dolt-server-cmd": "Every target and shard ran tests (cmd-dolt)", + } + + for _, mem := range bazelDoltIntMembers { + want := tagged[mem.tag] + if len(want) == 0 { + t.Fatalf("no BUILD rule is tagged %q", mem.tag) + } + tierName := bazelDoltIntTierNames[mem.tag] + + got := slices.Clone(gotTiers[tierName]) + slices.Sort(got) + if !slices.Equal(got, want) { + t.Errorf("classify step --tier %s=... labels %v, want the %q-tagged labels from the BUILD files %v", tierName, got, mem.tag, want) + } + + step := job.step(t, checkSteps[mem.tag]) + for _, label := range want { + if !strings.Contains(step.Run, "--label "+label) { + t.Errorf("%s step %q missing --label %s (one of %q's tagged labels)", bazelDoltIntJobName, checkSteps[mem.tag], label, mem.tag) + } + } + gotLabelCount := strings.Count(step.Run, "--label ") + if gotLabelCount != len(want) { + t.Errorf("%s step %q has %d --label args, want exactly %d (%v)", bazelDoltIntJobName, checkSteps[mem.tag], gotLabelCount, len(want), want) + } + } +} diff --git a/scripts/bazel_dolt_race_test.go b/scripts/bazel_dolt_race_test.go index 7d585164c..1225344ec 100644 --- a/scripts/bazel_dolt_race_test.go +++ b/scripts/bazel_dolt_race_test.go @@ -44,6 +44,22 @@ var bazelUnionLanes = map[string][]string{ bazelDoltRaceJobName: {bazelDoltJobName, bazelProxiedJobName}, } +// bazelUnionHasFlagGatedMember reports whether a union job runs in place of +// at least one flag-gated, advisory member (bazel-dolt-int's bazel-cmd-dolt, +// item 5 A2): such a union must carry that member's own continue-on-error, so +// a failure confined to that member's tier stays out of the call's aggregate +// result (BAZEL) in the union's mode exactly as it already is through the +// member outside it. The gated member's id still reads the union's result +// directly (bazelLaneReporter), unaffected either way. +func bazelUnionHasFlagGatedMember(job string) bool { + for _, m := range bazelUnionLanes[job] { + if _, ok := bazelFlagGatedLanes[m]; ok { + return true + } + } + return false +} + // bazelUnionOf: the union job a gated member belongs to, or "". func bazelUnionOf(member string) string { for union, members := range bazelUnionLanes { @@ -54,11 +70,23 @@ func bazelUnionOf(member string) string { return "" } +// bazelUnionMemberOutputKey: the union job's own output key that carries a +// member's tier result, for unions that do not report "result" uniformly +// (item 5 A2's bazel-dolt-int: must-fix 2's per-tier outputs, since a +// job-level continue-on-error would hide a server-storage failure from +// anything that is not itself gated on that one tier). Members not listed +// here read their union's plain "result" output, as dolt-race's do. +var bazelUnionMemberOutputKey = map[string]string{ + bazelServerJobName: "server_storage_result", + bazelCmdDoltJobName: "cmd_dolt_result", +} + // bazelLaneCallOutput: the workflow_call output value bazel.yml must declare // for a lane that reports its job.status. func bazelLaneCallOutput(lane string) string { if union := bazelUnionOf(lane); union != "" { - return "${{ jobs." + union + ".outputs.result || jobs." + lane + ".outputs.result }}" + key := cmp.Or(bazelUnionMemberOutputKey[lane], "result") + return "${{ jobs." + union + ".outputs." + key + " || jobs." + lane + ".outputs.result }}" } return "${{ jobs." + lane + ".outputs.result }}" } @@ -100,11 +128,13 @@ func bazelCallOutputValue(t *testing.T, name string, jobs map[string]string) str } ctx := map[string]string{} for _, m := range regexp.MustCompile(`jobs\.([A-Za-z0-9_-]+)\.outputs\.([A-Za-z0-9_-]+)`).FindAllStringSubmatch(expr, -1) { - v := "" - if m[2] == "result" { - v = jobs[m[1]] - } - ctx[m[0]] = v + // Every job.outputs reference in a lane's own workflow_call output + // expression carries that job's conclusion for gating purposes, + // whatever its output key is named (plain "result", or one of a + // union's per-tier keys: must-fix 2). jobs is keyed by job name only, + // so every reference to a given job resolves to the same simulated + // value regardless of which output key reads it. + ctx[m[0]] = jobs[m[1]] } v, err := evalGHExpr(expr, ctx) if err != nil { diff --git a/scripts/bazel_rrc_test.go b/scripts/bazel_rrc_test.go index d2dc07f13..90ff93cb9 100644 --- a/scripts/bazel_rrc_test.go +++ b/scripts/bazel_rrc_test.go @@ -49,9 +49,9 @@ const ( // alone under `canary`). Not bazel-release-cross (other target platforms, // not seeded) nor the package gates. var bazelRRCReadLanes = []string{ - "bazel-cmd-dolt", "bazel-dolt-race", "bazel-doltserver", "bazel-embedded", - "bazel-integration", "bazel-proxied", "bazel-pure", "bazel-server-storage", - "bazel-test", + "bazel-cmd-dolt", "bazel-dolt-int", "bazel-dolt-race", "bazel-doltserver", + "bazel-embedded", "bazel-integration", "bazel-proxied", "bazel-pure", + "bazel-server-storage", "bazel-test", } // bazelRRCReadLines: the reader's .bazelrc.local lines; both are key neutral diff --git a/scripts/ci_analytics_workflow_test.go b/scripts/ci_analytics_workflow_test.go index 5cf0878c4..a8b8828cd 100644 --- a/scripts/ci_analytics_workflow_test.go +++ b/scripts/ci_analytics_workflow_test.go @@ -25,6 +25,10 @@ var ciAnalyticsLaneJobs = map[string]string{ // The dolt-server + proxied union lane (item 5; mode remote): its // own lane value, so the A/B queries can pair it with its members. "bazel-dolt-race": "dolt-race", + // The server-Dolt storage + cmd/bd Dolt-server union lane (item 5 A2; + // mode remote): its own lane value, so the A/B queries can pair it + // with its members. + "bazel-dolt-int": "dolt-int", } // ciAnalyticsExcludedJobs are bazel.yml jobs that deliberately do not run diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index 28671e5e4..cf42da5fb 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -75,11 +75,12 @@ func bazelRRCCacheReaderRunsOn(label string) string { // isBazelRRCCacheReader: a remote repo contents cache reader // (bazelRRCReadLanes) that also runs in mode cache (not remote-only). -// bazel-dolt-race runs only in mode remote (its own if, -// TestBazelDoltRaceArmsExactlyOnePerMode), so it keeps the lanes' -// ubuntu-latest fallback like the remote-only jobs. +// bazel-dolt-race and bazel-dolt-int run only in mode remote (their own ifs, +// TestBazelDoltRaceArmsExactlyOnePerMode, TestBazelDoltIntArmsExactlyOnePerMode), +// so they keep the lanes' ubuntu-latest fallback like the remote-only jobs: +// the fallback branch of their runs-on ternary is never actually taken. func isBazelRRCCacheReader(name string) bool { - return slices.Contains(bazelRRCReadLanes, name) && !bazelRemoteOnlyJobs[name] && name != bazelDoltRaceJobName + return slices.Contains(bazelRRCReadLanes, name) && !bazelRemoteOnlyJobs[name] && name != bazelDoltRaceJobName && name != bazelDoltIntJobName } // The default lane size: every action executes on rbe-west, but the diff --git a/scripts/ci_merge_queue_test.go b/scripts/ci_merge_queue_test.go index f1813b326..dd88faaa0 100644 --- a/scripts/ci_merge_queue_test.go +++ b/scripts/ci_merge_queue_test.go @@ -208,6 +208,7 @@ func TestMergeQueuePullRequestFieldsHandleMergeGroup(t *testing.T) { "bazel-server-storage.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", "bazel-cmd-dolt.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", "bazel-dolt-race.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-dolt-int.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", } used := map[string]bool{} var fallbacks []string diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 846ea24c5..d284c8e9d 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1567,7 +1567,7 @@ const ( // bazel.yml's jobs: the rbe job that decides the execution mode, the // --config=ci lane, one job per CI job a Bazel config mirrors, and the two // package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings. -var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltRaceJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName, bazelRRCSeedJob, bazelRRCVerifyJob} +var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltIntJobName, bazelDoltRaceJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName, bazelRRCSeedJob, bazelRRCVerifyJob} // The lanes that only run remotely (skipped unless the rbe job chose remote); // bazel-integration runs remotely or with the read-only cache (bazelIntegIf); @@ -1638,6 +1638,51 @@ func ciAnalyticsContinueOnErrorPaths(t *testing.T) map[string]bool { return paths } +// bazelDoltIntTestContinueOnErrorPath: bazel-dolt-int's `bazel test` step's +// continue-on-error key path, one of the step-level exceptions +// TestBazelWorkflowSecretsAndFailureSurface allows (must-fix 2). Unlike the +// CI analytics steps' (reporting only, read by nothing), this one is load +// bearing: the always() "Classify tier results" step right after it (which +// itself carries no continue-on-error) reads the test step's outcome and rc +// output and decides job.status per-tier, so the test step's own +// continue-on-error does not hide a failure from pr.yml's ci-gate, it only +// moves which step's exit code determines it. Matched by job and step name +// (not a fixed index), since a future step added above it would shift it. +func bazelDoltIntTestContinueOnErrorPath(t *testing.T) string { + t.Helper() + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDoltIntJobName) + for i, step := range job.Steps { + if step.ID == "test" { + return fmt.Sprintf(".jobs.%s.steps[%d].continue-on-error", bazelDoltIntJobName, i) + } + } + t.Fatalf("%s has no step with id \"test\"", bazelDoltIntJobName) + return "" +} + +// bazelDoltIntCheckContinueOnErrorPaths: bazel-dolt-int's two per-tier +// check_testcases.py steps' continue-on-error key paths (should-fix 2, +// PR #7535 review), the same load-bearing shape as +// bazelDoltIntTestContinueOnErrorPath above: each step's own continue-on- +// error keeps a zero-testcase shard in one tier from failing the job +// directly, but the always() "Classify tier results" step folds its outcome +// into that tier's result through --check, so nothing is hidden from +// pr.yml's ci-gate, only moved. +func bazelDoltIntCheckContinueOnErrorPaths(t *testing.T) map[string]bool { + t.Helper() + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDoltIntJobName) + paths := map[string]bool{} + for i, step := range job.Steps { + if step.ID == "check-ss" || step.ID == "check-cd" { + paths[fmt.Sprintf(".jobs.%s.steps[%d].continue-on-error", bazelDoltIntJobName, i)] = true + } + } + if len(paths) != 2 { + t.Fatalf("%s: want 2 check-ss/check-cd steps, found %d", bazelDoltIntJobName, len(paths)) + } + return paths +} + // rbe-fork: the lanes' setup-bazel env that asks it for a certificate (modes // fork-ro and fork-rw only), and the mint's status URL the rbe job probes. const ( @@ -1726,8 +1771,15 @@ const bazelIntegIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outp // `integration` input (an explicit "off" skips it too, gastownhall/beads // #7486 review), but unlike bazel-integration/bazel-cmd-dolt it is // remote-only (wantRemoteOnlyIf), not mode cache -- there is no PR-time -// warm-cache run of this tier, only remote or nothing. -const bazelServerStorageIf = "${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }}" +// warm-cache run of this tier, only remote or nothing. Mode remote minus: +// there bazel-dolt-int runs this tier instead (item 5 A2, +// bazel_dolt_int_test.go). +const bazelServerStorageIf = "${{ needs.rbe.outputs.enabled == 'true' && needs.rbe.outputs.mode != 'remote' && inputs.integration != 'off' }}" + +// bazelCmdDoltMemberIf: bazel-cmd-dolt's pre-union rule (bazelIntegIf) minus +// mode remote, where bazel-dolt-int runs this tier instead (item 5 A2, +// bazel_dolt_int_test.go). +const bazelCmdDoltMemberIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' && needs.rbe.outputs.mode != 'remote' }}" // F3: package-mcp and package-npm's if. Unlike every other lane's, it does // not read the rbe job's outputs at all - only the caller's package-gates @@ -1881,7 +1933,22 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { // bazel.yml's own run - and thus pr.yml's ci-gate - red for a job // nothing requires. _, flagGated := bazelFlagGatedLanes[name] - if job.ContinueOnError && name != bazelRBEPrewarmJobName && !flagGated { + // A union lane with a flag-gated member (item 5's bazel-dolt-race, + // standing in for bazel-doltserver and bazel-proxied in mode remote; + // see bazelUnionLanes) carries the same job-level continue-on-error + // as that member would alone: the gated member's id still reads this + // job's result directly (unaffected), and continue-on-error only + // keeps a flag-gated-member-only failure out of the call's own + // aggregate (BAZEL), as it already is through the member outside + // this mode. This is unrelated to item 5 A2's bazel-dolt-int, which + // carries no job-level continue-on-error at all: its gating is + // entirely step-level (its own `bazel test` step, and its two + // per-tier check_testcases.py steps, each folded into job.status by + // the always() classify step right after them; see + // bazelDoltIntTestContinueOnErrorPath and + // bazelDoltIntCheckContinueOnErrorPaths in + // TestBazelWorkflowSecretsAndFailureSurface, must-fix 2). + if job.ContinueOnError && name != bazelRBEPrewarmJobName && !flagGated && !bazelUnionHasFlagGatedMember(name) { t.Errorf("%s continue-on-error hides failures from pr.yml's ci-gate", name) } // A flag-gated lane is advisory until its flag says otherwise, and @@ -1977,10 +2044,15 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { // review). wantJobIf = bazelServerStorageIf } - } else if name == bazelIntegJobName || name == bazelCmdDoltJobName { - // The cmd/bd Dolt-server tier runs exactly where the integration - // lane does: its build is --config=integration's. + } else if name == bazelIntegJobName { + // bazel-integration is not part of either union; it stays as + // before, in every mode the union jobs also run in. wantJobIf = bazelIntegIf + } else if name == bazelCmdDoltJobName { + // The cmd/bd Dolt-server tier runs exactly where the integration + // lane does, minus mode remote: there bazel-dolt-int runs this + // tier instead (item 5 A2, bazel_dolt_int_test.go). + wantJobIf = bazelCmdDoltMemberIf } else if name == bazelRBEPrewarmJobName { // The lanes' runs-on ternary at 2 vCPU (no Bazel client); only // the size and the if differ. Not a lane: never uses @@ -1992,9 +2064,15 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { // (bazel_dolt_race_test.go); the lanes' runner and setup-bazel // env otherwise. wantJobIf = bazelDoltRaceJobIf + } else if name == bazelDoltIntJobName { + // Mode remote only, in place of its members + // (bazel_dolt_int_test.go); the lanes' runner and setup-bazel + // env otherwise. + wantJobIf = bazelDoltIntJobIf } - // The union's members: their own rule, minus mode remote - // (TestBazelDoltRaceArmsExactlyOnePerMode). + // The unions' members: their own rule, minus mode remote + // (TestBazelDoltRaceArmsExactlyOnePerMode, + // TestBazelDoltIntArmsExactlyOnePerMode). switch name { case bazelDoltJobName: wantJobIf = bazelDoltMemberIf @@ -2555,14 +2633,21 @@ func TestBazelLaneIsGated(t *testing.T) { t.Errorf("%s job %s: gated=%v advisory=%v flag-gated=%v union=%v; add it to exactly one of bazelLaneGateIDs (with a CI_GATE_REQUIRED id in pr.yml), bazelFlagGatedLanes (with its pr.yml flag), bazelUnionLanes (with the gated members it runs in place of) or bazelAdvisoryLanes (with the reason)", bazelWorkflowName, name, gated, advisory, flagGated, union) } - // Each lane reports its job.status from an always() last step. - if !reflect.DeepEqual(job.Outputs, map[string]string{"result": "${{ steps.result.outputs.result }}"}) { - t.Errorf("%s outputs = %v, want only result from the result step", name, job.Outputs) - } - last := job.Steps[len(job.Steps)-1] - if last.ID != "result" || last.If != "${{ always() }}" || !reflect.DeepEqual(last.Env, map[string]string{"JOB_STATUS": "${{ job.status }}"}) || - last.Run != `echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT"` { - t.Errorf("%s last step = %+v; want the always() job.status recorder", name, last) + // Each lane reports its job.status from an always() last step, except + // bazel-dolt-int: must-fix 2's per-tier outputs mean job.status + // itself is not what either member reads (a cmd-dolt-only failure + // must leave job.status green while cmd_dolt_result reads failure), + // so it has its own shape, pinned in full by + // TestBazelDoltIntIsGated instead. + if name != bazelDoltIntJobName { + if !reflect.DeepEqual(job.Outputs, map[string]string{"result": "${{ steps.result.outputs.result }}"}) { + t.Errorf("%s outputs = %v, want only result from the result step", name, job.Outputs) + } + last := job.Steps[len(job.Steps)-1] + if last.ID != "result" || last.If != "${{ always() }}" || !reflect.DeepEqual(last.Env, map[string]string{"JOB_STATUS": "${{ job.status }}"}) || + last.Run != `echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT"` { + t.Errorf("%s last step = %+v; want the always() job.status recorder", name, last) + } } if !union { wantCallOutputs[name] = bazelLaneCallOutput(name) @@ -2573,8 +2658,10 @@ func TestBazelLaneIsGated(t *testing.T) { t.Errorf("bazelUnionLanes lists %s, which %s does not have", union, bazelWorkflowName) } for _, m := range members { - if _, gated := bazelLaneGateIDs[m]; !gated { - t.Errorf("union lane %s member %s is not a gated lane", union, m) + _, gated := bazelLaneGateIDs[m] + _, flagGated := bazelFlagGatedLanes[m] + if !gated && !flagGated { + t.Errorf("union lane %s member %s is neither a gated nor a flag-gated lane", union, m) } } } @@ -2880,11 +2967,20 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string case bazelServerStorageIf: // Remote-only (unlike bazelIntegIf, no mode cache run), but also // honors `integration` like bazel-integration/bazel-cmd-dolt - // (gastownhall/beads#7486 review). + // (gastownhall/beads#7486 review). Minus mode remote: there + // bazel-dolt-int runs this tier instead (item 5 A2). if strings.EqualFold(with["integration"], "off") { return map[string]bool{} } - return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true} + return map[string]bool{"fork-ro": true, "fork-rw": true} + case bazelCmdDoltMemberIf: + // Where bazel-integration runs, minus mode remote: there + // bazel-dolt-int runs this tier instead (item 5 A2, + // bazel_dolt_int_test.go). + if strings.EqualFold(with["integration"], "off") { + return map[string]bool{} + } + return map[string]bool{"fork-ro": true, "fork-rw": true, "cache": true} case bazelPackageGatesIf: // F3: unlike every other lane, these two never skip because of the // rbe job's decision - only because the caller turned them off @@ -2906,6 +3002,18 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string // nightly check (schedule, dispatch): never in a pull_request, // merge_group or pull_request_target call, in any mode. return map[string]bool{} + case bazelDoltIntJobIf: + // Mode remote only (bazel-dolt-race's own if is the same shape), and + // also honors `integration` like its members (#7491's review, + // must-fix 1): a server-storage/cmd-dolt skip via integration="off" + // must not become a silent "nothing ran the tier" gap when mode is + // remote (bazel_dolt_int_test.go's + // TestBazelDoltIntArmsExactlyOnePerMode checks that directly; this + // case just keeps bazelLaneRunModes consistent with it). + if strings.EqualFold(with["integration"], "off") { + return map[string]bool{} + } + return map[string]bool{"remote": true} } t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr) return nil @@ -3073,9 +3181,11 @@ func TestBazelGateSimulation(t *testing.T) { if bazelLaneReporter(lanes, lane, mode) == "" { wantSkips = append(wantSkips, id) } - } else if g, flagGated := bazelFlagGatedLanes[lane]; !modes[mode] && flagGated { + } else if g, flagGated := bazelFlagGatedLanes[lane]; flagGated && bazelLaneReporter(lanes, lane, mode) == "" { // Listed even while advisory, so flipping the flag is the - // only change that makes it required. + // only change that makes it required. Skipped by design only + // when neither the lane nor its union lane (bazel-dolt-int in + // mode remote, for bazel-cmd-dolt) runs in this mode. wantSkips = append(wantSkips, g.id) } } @@ -3153,7 +3263,19 @@ func TestBazelGateSimulation(t *testing.T) { // Required (flag "true"): exactly a gated lane. scFlags(lane+" required, as designed", "success", base(), on, true, "") if !modes[mode] { - scFlags(lane+" required, ran and failed", "success", with(lane, "failure"), on, false, g.id) + // Not its own job in this mode (skipped by design, or its + // union reports instead, as bazel-dolt-int does for + // bazel-cmd-dolt in mode remote); if it ran anyway and + // failed, the gate still sees it through its id, unless + // the union's result shadows the stray failure (reported + // first in the workflow_call output), in which case + // there is nothing to blame it for. + out := with(lane, "failure") + if bazelCallOutputValue(t, lane, out) != "failure" { + scFlags(lane+" required, ran and failed", "success", out, on, true, "") + } else { + scFlags(lane+" required, ran and failed", "success", out, on, false, g.id) + } continue } for _, result := range []string{"", "failure", "cancelled"} { @@ -4609,6 +4731,8 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { } } ciAnalyticsPaths := ciAnalyticsContinueOnErrorPaths(t) + doltIntTestPath := bazelDoltIntTestContinueOnErrorPath(t) + doltIntCheckPaths := bazelDoltIntCheckContinueOnErrorPaths(t) secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { if key && value == "continue-on-error" { @@ -4622,9 +4746,19 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { // two CI analytics steps in every lane job // (bazelCIAnalyticsStepNames): reporting only, never a `needs` // of anything and never read back by this workflow or its - // callers. - _, flagGated := bazelFlagGatedLanes[strings.TrimSuffix(strings.TrimPrefix(path, ".jobs."), ".continue-on-error")] - if path != bazelRBEPrewarmContinueOnErrorPath && !flagGated && !ciAnalyticsPaths[path] { + // callers. So are three steps in bazel-dolt-int's own job: its + // `bazel test` step (bazelDoltIntTestContinueOnErrorPath) and its + // two per-tier "Every target and shard ran tests" steps, + // check-ss and check-cd (bazelDoltIntCheckContinueOnErrorPaths, + // should-fix 2) — each one's own continue-on-error keeps a + // single shard's failure from failing the job directly, but the + // always() classify step right after them (no continue-on-error + // of its own) folds every one of their outcomes, plus the test + // step's own rc, into job.status per tier via --check, so + // nothing is hidden from pr.yml's ci-gate, only moved. + jobName := strings.TrimSuffix(strings.TrimPrefix(path, ".jobs."), ".continue-on-error") + _, flagGated := bazelFlagGatedLanes[jobName] + if path != bazelRBEPrewarmContinueOnErrorPath && path != doltIntTestPath && !doltIntCheckPaths[path] && !flagGated && !ciAnalyticsPaths[path] && !bazelUnionHasFlagGatedMember(jobName) { t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path) } } @@ -5862,9 +5996,7 @@ func TestBazelCmdDoltJob(t *testing.T) { "test:integration --@rules_go//go/config:race", "test:" + config + " --test_tag_filters=dolt-server-cmd", "test:" + config + " --test_timeout=-1,-1,-1,1200", - "test:" + config + " --test_arg=-test.timeout=19m", "test:" + config + " --test_arg=-test.parallel=4", - "test:" + config + " --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1", "test:" + config + " --test_env=GO_TEST_WRAP_TESTV=1", } { if !rc[want] { @@ -5909,10 +6041,18 @@ func TestBazelCmdDoltJob(t *testing.T) { } for _, want := range []string{ `srcs = ["//tools/bazel:go_test_pinned_shard.sh"]`, - "args = [\n \"$(rootpath :dolt_server_pinned_shards.txt)\",\n \"$(rootpath :bd_test)\",\n ]", + `"$(rootpath :dolt_server_pinned_shards.txt)",`, + `"$(rootpath :bd_test)",`, + // Item 5 A2: moved here from --config=doltserver-cmd's --test_arg, + // so --config=dolt-int does not set it for server-storage's targets + // too (whose own Go timeout is 15m). + `"-test.timeout=19m",`, `tags = ["dolt-server-cmd"]`, `"GOMAXPROCS": "4"`, `"BEADS_TEST_GIT_IDENTITY": "1"`, + // Item 5 A2: moved here from --config=doltserver-cmd's --test_env, + // same reason as the args move above. + `"BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION": "1",`, } { if !strings.Contains(rule, want) { t.Errorf("%s lacks %s", target, want) @@ -5990,11 +6130,19 @@ func TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers(t *testing.T) { } } - // Coverage where it is dropped: bazel-cmd-dolt runs in every mode - // bazel-integration does (same if, for every caller's inputs). - if integ.If != bazelIntegIf || cmdDolt.If != integ.If { - t.Errorf("%s if = %q, %s if = %q; want both %q (the cmd/bd tier must run wherever integration drops bd_test)", - bazelIntegJobName, integ.If, bazelCmdDoltJobName, cmdDolt.If, bazelIntegIf) + // Coverage where it is dropped: the cmd/bd tier (bazel-cmd-dolt, or + // bazel-dolt-int in mode remote, item 5 A2) runs in every mode + // bazel-integration does. + if integ.If != bazelIntegIf { + t.Errorf("%s if = %q, want %q", bazelIntegJobName, integ.If, bazelIntegIf) + } + wantModes := bazelLaneRunModes(t, bazelIntegJobName, bazelIntegIf, nil) + cmdDoltModes := bazelLaneRunModes(t, bazelCmdDoltJobName, cmdDolt.If, nil) + doltIntModes := bazelLaneRunModes(t, bazelDoltIntJobName, workflow.job(t, bazelDoltIntJobName).If, nil) + for _, mode := range bazelRBEModes { + if got := cmdDoltModes[mode] || doltIntModes[mode]; got != wantModes[mode] { + t.Errorf("mode %s: cmd/bd tier covered (bazel-cmd-dolt or bazel-dolt-int) = %v, want %v (the cmd/bd tier must run wherever integration drops bd_test)", mode, got, wantModes[mode]) + } } // ... and pr.yml's gate accepts their skips in the same modes only. requireHostTool(t, "bash") diff --git a/scripts/nogo_lint_policy_test.go b/scripts/nogo_lint_policy_test.go index e1c56bfbf..39bdfa06c 100644 --- a/scripts/nogo_lint_policy_test.go +++ b/scripts/nogo_lint_policy_test.go @@ -410,9 +410,9 @@ var nogoConfigurations = []nogoConfigurationRow{ }, { name: "integ", - configs: []string{"integration", "doltserver-integration", "doltserver-cmd"}, + configs: []string{"integration", "doltserver-integration", "doltserver-cmd", "dolt-int"}, owner: bazelIntegJobName, - nonOwners: []string{bazelServerJobName, bazelCmdDoltJobName}, + nonOwners: []string{bazelServerJobName, bazelCmdDoltJobName, bazelDoltIntJobName}, ownerIsNogoOnlyStep: true, nogoOnlyStepTargets: []string{"//internal/testutil/integration:integration"}, }, diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 7509c82eb..304022e15 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -1173,6 +1173,10 @@ func TestBazelRetiredLanesArePinned(t *testing.T) { // The dolt-race union config carries its proxied member's // line (TestBazelDoltRaceConfigIsMemberUnion pins the union). evictionAllowed["test:"+bazelDoltRaceConfig+" --experimental_remote_cache_eviction_retries=0"] = true + // The dolt-int union config carries its server-storage member's line + // (item 5 A2: doltserver-cmd has none of its own; + // TestBazelDoltIntConfigIsMemberUnion pins the union). + evictionAllowed["test:"+bazelDoltIntConfig+" --experimental_remote_cache_eviction_retries=0"] = true for _, line := range strings.Split(rc, "\n") { line = strings.TrimSpace(line) if !strings.HasPrefix(line, "#") && strings.Contains(line, "remote_cache_eviction_retries") && !evictionAllowed[line] { diff --git a/tests/regression/BUILD.bazel b/tests/regression/BUILD.bazel index 2c1f527f2..a1f987912 100644 --- a/tests/regression/BUILD.bazel +++ b/tests/regression/BUILD.bazel @@ -38,8 +38,10 @@ go_test( # Tags: # dolt-server-cmd: starts a hermetic dolt sql-server and needs the # integration build tag; selected by --config=doltserver-cmd (bazel.yml's -# bazel-cmd-dolt lane) and excluded from every other lane, whose -# configurations either drop both test files or set BEADS_TEST_SKIP=dolt. +# bazel-cmd-dolt lane) and, in mode remote, by --config=dolt-int +# (bazel-dolt-int, item 5 A2's union of that lane and bazel-server-storage) +# and excluded from every other lane, whose configurations either drop +# both test files or set BEADS_TEST_SKIP=dolt. sh_test( name = "regression_test", size = "large", @@ -50,6 +52,10 @@ sh_test( args = [ "$(rootpath :pinned_shards.txt)", "$(rootpath :regression_suite)", + # Item 5 A2: moved from --config=doltserver-cmd's --test_arg (see + # cmd/bd:bd_dolt_server_test's comment); keeps --config=dolt-int's + # other member (server-storage, a 15m Go timeout) unaffected. + "-test.timeout=19m", ], data = [ "BASELINE_VERSION", @@ -62,6 +68,8 @@ sh_test( "BD_REGRESSION_BASELINE_BIN": "$(rlocationpath @bd_releases//:v0.49.6)", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_DOLT_SERVER": "local", + # Item 5 A2: moved from --config=doltserver-cmd's --test_env. + "BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION": "1", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", }, shard_count = 14, diff --git a/tools/bazel/check_testcases.py b/tools/bazel/check_testcases.py index 10cabcc35..e00bbc72f 100755 --- a/tools/bazel/check_testcases.py +++ b/tools/bazel/check_testcases.py @@ -2,7 +2,7 @@ """Fail if a Bazel test (or any shard of it) ran no Go tests. Usage: check_testcases.py --bep [--testlogs DIR] - [--not-go LABEL ...] + [--not-go LABEL ...] [--label LABEL ...] A test target whose Go binary matches no test still exits 0: a shard script whose manifest assigns a shard nothing ("no tests assigned"), a -test.run @@ -22,8 +22,19 @@ cannot linger. //pkg:all exempts every test in that package (one that holds only non-Go tests, such as //scripts/repochecks), and must match at least one tested target. +--label LABEL (repeatable) restricts which tested targets this invocation +judges, by exact label; a requested label absent from the BEP is itself a +failure (a typo or a stale label must not silently narrow the check to +nothing). item 5 A2: bazel-dolt-int's union BEP has both tiers' targets, and +its two "Every target and shard ran tests ()" steps each pass only +their own tier's labels, so a cmd-dolt zero-testcase shard reaches only +cmd_dolt_result (classify_tier_results.py's --check), not server-storage's, +and vice versa. Omitted (bazel-test and every other lane's own step, +unchanged): every tested target in the BEP, as before. + Exit status: 0 if every test.xml lists a test, 1 otherwise (including a BEP -with no test results at all). +with no test results at all, or a --label naming a target the BEP has no +result for). """ import argparse @@ -45,9 +56,18 @@ def count_testcases(xml_path): return n -def check(tested, testlogs, not_go=()): - """Return ([(xml path, count)], [problems]) for {label: shard_count}.""" +def check(tested, testlogs, not_go=(), labels=None): + """Return ([(xml path, count)], [problems]) for {label: shard_count}. + + labels, if given (a set), restricts the check to those exact labels; a + label with no entry in tested is itself a problem (see the module + docstring's --label section). + """ counts, problems = [], [] + if labels is not None: + for label in sorted(set(labels) - set(tested)): + problems.append(f"{label}: --label, but the BEP has no result for it") + tested = {l: n for l, n in tested.items() if l in labels} if not tested: problems.append("the BEP lists no test results") def exempt(label, entry): @@ -83,11 +103,13 @@ def main(argv=None): ap.add_argument("--testlogs", default=None, help="default: from the BEP, else ./bazel-testlogs") ap.add_argument("--not-go", action="append", default=[], metavar="LABEL", help="a tested non-Go target to leave out of the count (repeatable)") + ap.add_argument("--label", action="append", default=None, metavar="LABEL", + help="restrict to this exact target label (repeatable); default: every tested target in the BEP") args = ap.parse_args(argv) tested, _, bep_testlogs = read_bep(args.bep) testlogs = args.testlogs or bep_testlogs or "bazel-testlogs" - counts, problems = check(tested, testlogs, set(args.not_go)) + counts, problems = check(tested, testlogs, set(args.not_go), labels=set(args.label) if args.label else None) for rel, n in counts: print(f"{n:5d} {rel}") total = sum(n for _, n in counts) diff --git a/tools/bazel/check_testcases_test.py b/tools/bazel/check_testcases_test.py new file mode 100644 index 000000000..9348a81a6 --- /dev/null +++ b/tools/bazel/check_testcases_test.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Unit tests for check_testcases.py, in particular its --label flag. + +item 5 A2: bazel-dolt-int's union BEP has both tiers' targets, so its two +"Every target and shard ran tests ()" steps each need --label to scope +the check to their own tier's labels only (should-fix 2, PR #7535 review). +This covers check()'s --label behaviour directly; main()'s argument wiring is +exercised indirectly by bazel_dolt_int_test.go's step-content assertions and +by the manual BEP runs in that PR's review thread. +""" + +import os +import sys +import tempfile +import unittest +import xml.etree.ElementTree as ET + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from check_testcases import check # noqa: E402 + + +def write_testxml(path, testcase_names): + root = ET.Element("testsuite") + for name in testcase_names: + ET.SubElement(root, "testcase", name=name) + ET.ElementTree(root).write(path) + + +class CheckTestcasesTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.testlogs = self.tmp.name + + def tearDown(self): + self.tmp.cleanup() + + def write(self, label, names, shard=None): + rel_dir = label.lstrip("/").replace(":", "/") + d = os.path.join(self.testlogs, rel_dir) + if shard is not None: + d = os.path.join(d, "shard_%d_of_1" % shard) + os.makedirs(d, exist_ok=True) + write_testxml(os.path.join(d, "test.xml"), names) + + def test_no_label_checks_every_tested_target(self): + self.write("//a:x", ["TestFoo"]) + self.write("//b:y", ["TestBar"]) + counts, problems = check({"//a:x": 1, "//b:y": 1}, self.testlogs) + self.assertEqual(problems, []) + self.assertEqual(len(counts), 2) + + def test_label_restricts_to_named_targets(self): + self.write("//a:x", ["TestFoo"]) + self.write("//b:y", []) # would fail if checked + counts, problems = check( + {"//a:x": 1, "//b:y": 1}, self.testlogs, labels={"//a:x"} + ) + self.assertEqual(problems, []) + self.assertEqual(len(counts), 1) + + def test_label_not_in_bep_is_a_failure(self): + self.write("//a:x", ["TestFoo"]) + counts, problems = check( + {"//a:x": 1}, self.testlogs, labels={"//a:x", "//missing:z"} + ) + self.assertTrue( + any("//missing:z" in p and "--label" in p for p in problems), + problems, + ) + + def test_label_does_not_mask_a_zero_testcase_shard_in_its_own_tier(self): + self.write("//a:x", []) + counts, problems = check({"//a:x": 1}, self.testlogs, labels={"//a:x"}) + self.assertTrue(any("//a:x" in p and "lists no tests" in p for p in problems)) + + def test_label_scoping_is_independent_of_other_tiers_zero_testcase_shard(self): + # The exact bazel-dolt-int scenario: cmd-dolt's zero-testcase shard + # must not surface when --label scopes the check to server-storage. + self.write("//server:ss", ["TestServer"]) + self.write("//cmd:cd", []) + counts, problems = check( + {"//server:ss": 1, "//cmd:cd": 1}, + self.testlogs, + labels={"//server:ss"}, + ) + self.assertEqual(problems, []) + self.assertEqual(len(counts), 1) + + def test_empty_label_set_with_no_tested_targets_is_a_failure(self): + counts, problems = check({}, self.testlogs, labels={"//a:x"}) + self.assertTrue(any("//a:x" in p for p in problems)) + + def test_not_go_still_applies_within_a_labeled_subset(self): + self.write("//a:x", ["TestFoo"]) + self.write("//a:shtest", []) # exempt via --not-go + counts, problems = check( + {"//a:x": 1, "//a:shtest": 1}, + self.testlogs, + not_go=("//a:shtest",), + labels={"//a:x", "//a:shtest"}, + ) + self.assertEqual(problems, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/bazel/classify_tier_results.py b/tools/bazel/classify_tier_results.py new file mode 100644 index 000000000..97733483e --- /dev/null +++ b/tools/bazel/classify_tier_results.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Classify a union `bazel test` run's result by tier, from its BEP. + +Usage: classify_tier_results.py --bep --rc + --tier NAME=LABEL[,LABEL...] [--tier NAME=LABEL...] + --gate-tier NAME --github-output + [--check NAME=success|failure|cancelled ...] + +item 5 A2's bazel-dolt-int job runs two tiers (server-storage, cmd-dolt) in +one `bazel test --config=dolt-int` invocation. The job carries no job-level +continue-on-error: on push, nightly and dispatch, nothing reads +BAZEL_SERVER_STORAGE off a `workflow_call` output, so a job-level +continue-on-error (reading job.status as success regardless of the `bazel +test` step's own outcome) would let a server-storage failure end the run +green there, where it must stay red exactly as it would on main today. The +`bazel test` step instead carries continue-on-error, and this script -- an +always() step of its own -- computes each tier's result independently from +the BEP and decides whether the job itself must fail. + +Bazel's own exit code governs first. 0 (ran, all passed) and 3 (ran, some +test failed) are the only codes this script trusts the BEP for: a build +failure, an infra error or a crash (any other code) cannot be safely +attributed to one tier's tests, so both tiers are marked "failure" and the +script always exits 1 for it (no silent cmd-dolt-only downgrade of a +not-actually-a-test-failure). A missing or non-integer --rc (bee-ghosttrack's +review, PR #7535 should-fix 2: the `bazel test` step's 25-minute timeout or a +cancellation before it can `echo "rc=$rc"` leaves bazel.yml's $TEST_RC empty) +takes this same branch rather than crashing argparse with no outputs +written: it is treated exactly like an exit code outside {0, 3}. + +Otherwise each --tier is "failure" iff a BEP testResult for one of its +labels has a non-PASSED status (FAILED, TIMEOUT, REMOTE_FAILURE, INCOMPLETE, +FLAKY all count: these lanes run flaky=False, so a flaky-looking status is +still a failure), or if the BEP has no result at all for one of its labels +(an attempt that never ran cannot be presumed to have passed; a genuine +coverage gap already fails the job through check_testcases.py's own step). + +Fail-closed on an unknown label (coordinator review, PR #7535 must-fix 1): +if the BEP has a non-PASSED result for any label that is not one of the +--tier lists (a newly `dolt-server-integration`/`dolt-server-cmd`-tagged +target added without updating the step's --tier args, i.e. bazel.yml and +this script's lists drifting from the BUILD files' tag sets -- see +TestBazelDoltIntTierLabelsMatchTheTags), every tier is marked "failure" and +the script exits 1: an un-enumerated failing target must not read as success +just because no --tier list claims it. + +--check NAME=OUTCOME (repeatable) folds a per-tier shell check's own GitHub +Actions step outcome into that tier's result: "success" leaves the BEP-only +result alone, anything else (bazel.yml passes the step's own +`steps..outcome`: "failure" or "cancelled") forces that tier to +"failure" regardless of the BEP. This is how bazel.yml's two +"Every target and shard ran tests ()" steps (the shared +check_testcases.py split per tier, PR #7535 should-fix 2) report a zero- +testcase shard without going through job.status: cmd-dolt's check runs with +its own continue-on-error so it cannot fail the job directly, but its result +must still reach cmd_dolt_result. + +Exit status is 1 iff --gate-tier's tier is "failure" (or the exit-code or +unknown-label branches above applied); a failure confined to another tier +leaves this script's own exit 0, so the job step -- and so job.status -- +stays green, mirroring that tier's own advisory, continue-on-error nature +outside mode remote. Every tier's result, win or lose, is appended to +--github-output as "_result=success|failure" (NAME lowercased, "-" +turned into "_"), so the job can expose one output per tier. + +Every flag may be given at most once, except --tier and --check (one per +distinct NAME: a repeated NAME, or an unrecognized flag, is a hard error -- +coordinator review, PR #7535 must-fix 3 -- so a future edit cannot silently +widen or narrow the gate by appending an extra --tier/--gate-tier/--check +that argparse would otherwise just let the last value win). +""" + +import argparse +import json +import sys + +PASS_STATUS = "PASSED" + + +class _StoreOnce(argparse.Action): + """Like the default "store" action, but errors if given more than once.""" + + def __call__(self, parser, namespace, values, option_string=None): + if getattr(namespace, self.dest, None) is not None: + parser.error(f"{option_string} may only be given once") + setattr(namespace, self.dest, values) + + +def read_test_statuses(bep_path): + """Return {label: set(status seen across every testResult event)}.""" + statuses = {} + with open(bep_path, encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + try: + ev = json.loads(line) + except ValueError: + continue + tr_id = ev.get("id", {}).get("testResult") + tr = ev.get("testResult") + if not tr_id or not tr: + continue + label = tr_id["label"] + statuses.setdefault(label, set()).add(tr.get("status", "NO_STATUS")) + return statuses + + +def tier_result(statuses, labels): + """Return "success" or "failure" for one tier's labels.""" + for label in labels: + seen = statuses.get(label) + if not seen or seen - {PASS_STATUS}: + return "failure" + return "success" + + +def output_key(name): + return name.lower().replace("-", "_") + "_result" + + +def classify(bep_path, rc, tiers, gate_tier, checks=None): + """Return ({tier: "success"|"failure"}, exit_status, reason|None). + + checks, if given, is {tier name: GitHub Actions step outcome}: any value + other than "success" forces that tier to "failure" (see the module + docstring's --check section). reason is None iff exit_status is 0. + """ + checks = checks or {} + try: + rc = int(rc) + except (TypeError, ValueError): + reason = ( + f"bazel exit code {rc!r} is missing or not an integer: the test " + "step likely timed out or was cancelled before it could record one" + ) + return {name: "failure" for name in tiers}, 1, reason + if rc not in (0, 3): + reason = f"bazel exit {rc} (not 0 or 3): a build or infra failure, not attributable to one tier" + return {name: "failure" for name in tiers}, 1, reason + + statuses = read_test_statuses(bep_path) + + known_labels = set() + for labels in tiers.values(): + known_labels.update(labels) + unknown_failed = sorted( + label for label, seen in statuses.items() + if label not in known_labels and (seen - {PASS_STATUS}) + ) + if unknown_failed: + # A real test failure Bazel reported (rc 3) that no --tier list + # claims: the tier/tag lists have drifted from what actually ran. + # Fail everything shut rather than silently reading it as success. + reason = f"non-PASSED result for a label outside every --tier: {', '.join(unknown_failed)}" + return {name: "failure" for name in tiers}, 1, reason + + results = {} + for name, labels in tiers.items(): + result = tier_result(statuses, labels) + if checks.get(name, "success") != "success": + result = "failure" + results[name] = result + if results.get(gate_tier) == "failure": + return results, 1, f"{gate_tier} tier failed" + return results, 0, None + + +def parse_tier_arg(spec): + name, _, labels = spec.partition("=") + if not name or not labels: + raise argparse.ArgumentTypeError(f"--tier {spec!r}: want NAME=LABEL[,LABEL...]") + return name, [l for l in labels.split(",") if l] + + +def parse_check_arg(spec): + name, sep, outcome = spec.partition("=") + if not name or not sep or not outcome: + raise argparse.ArgumentTypeError(f"--check {spec!r}: want NAME=OUTCOME") + return name, outcome + + +def _dict_no_duplicates(parser, pairs, flag): + out = {} + for name, value in pairs: + if name in out: + parser.error(f"{flag} {name!r} given more than once") + out[name] = value + return out + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--bep", required=True, action=_StoreOnce) + # Not type=int: an empty or non-integer --rc (the test step's own + # $TEST_RC left unset by a timeout or cancellation) must reach + # classify()'s own handling below, not crash argparse with no outputs + # written (should-fix 2). + p.add_argument("--rc", required=True, action=_StoreOnce) + p.add_argument("--tier", action="append", type=parse_tier_arg, default=[], dest="tiers", metavar="NAME=LABEL[,LABEL...]") + p.add_argument("--gate-tier", required=True, action=_StoreOnce) + p.add_argument("--github-output", required=True, action=_StoreOnce) + p.add_argument("--check", action="append", type=parse_check_arg, default=[], dest="checks", metavar="NAME=OUTCOME") + args = p.parse_args(argv) + + tiers = _dict_no_duplicates(p, args.tiers, "--tier") + if not tiers: + p.error("at least one --tier is required") + if args.gate_tier not in tiers: + p.error(f"--gate-tier {args.gate_tier!r} is not one of --tier's names {sorted(tiers)}") + checks = _dict_no_duplicates(p, args.checks, "--check") + for name in checks: + if name not in tiers: + p.error(f"--check {name!r} is not one of --tier's names {sorted(tiers)}") + + results, status, reason = classify(args.bep, args.rc, tiers, args.gate_tier, checks) + + with open(args.github_output, "a", encoding="utf-8") as f: + for name, result in results.items(): + line = f"{output_key(name)}={result}" + print(line) + f.write(line + "\n") + + if status: + print(f"::error::classify_tier_results: {reason}", file=sys.stderr) + return status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/classify_tier_results_test.py b/tools/bazel/classify_tier_results_test.py new file mode 100644 index 000000000..289b1c7e1 --- /dev/null +++ b/tools/bazel/classify_tier_results_test.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +"""Unit tests for classify_tier_results.py. + +Run directly: python3 tools/bazel/classify_tier_results_test.py +(or: python3 -m unittest tools.bazel.classify_tier_results_test, from the repo root) +""" + +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import classify_tier_results as ctr # noqa: E402 + +SS = "//internal/storage/dolt:dolt_server_full_test" +SS2 = "//internal/storage/dolt:dolt_server_conformance_test" +CD = "//cmd/bd:bd_dolt_server_test" +CD2 = "//tests/regression:regression_test" +UNKNOWN = "//internal/storage/dolt:dolt_server_new_test" + +TIERS = {"server-storage": [SS, SS2], "cmd-dolt": [CD, CD2]} + + +def write_bep(events): + f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8") + for ev in events: + f.write(json.dumps(ev) + "\n") + f.close() + return f.name + + +def result_event(label, status): + return {"id": {"testResult": {"label": label}}, "testResult": {"status": status}} + + +def all_passed_bep(extra=()): + return write_bep([result_event(l, "PASSED") for l in (SS, SS2, CD, CD2)] + list(extra)) + + +class TierResultTest(unittest.TestCase): + def test_all_passed_is_success(self): + statuses = {SS: {"PASSED"}, SS2: {"PASSED"}} + self.assertEqual(ctr.tier_result(statuses, [SS, SS2]), "success") + + def test_any_failed_label_is_failure(self): + statuses = {SS: {"PASSED"}, SS2: {"FAILED"}} + self.assertEqual(ctr.tier_result(statuses, [SS, SS2]), "failure") + + def test_missing_label_is_failure(self): + statuses = {SS: {"PASSED"}} + self.assertEqual(ctr.tier_result(statuses, [SS, SS2]), "failure") + + def test_flaky_status_is_failure(self): + # These lanes run flaky=False: a FLAKY-looking status is still a + # failure, never silently treated as a pass. + statuses = {SS: {"PASSED", "FLAKY"}} + self.assertEqual(ctr.tier_result(statuses, [SS]), "failure") + + def test_omitted_status_is_failure(self): + # proto3 JSON omits a NO_STATUS (0) field entirely; read_test_statuses + # falls back to the literal string "NO_STATUS", which is not PASSED. + statuses = {SS: {"NO_STATUS"}} + self.assertEqual(ctr.tier_result(statuses, [SS]), "failure") + + +class ClassifyTest(unittest.TestCase): + def test_rc_0_both_tiers_succeed(self): + path = all_passed_bep() + try: + results, status, reason = ctr.classify(path, 0, TIERS, "server-storage") + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "success", "cmd-dolt": "success"}) + self.assertEqual(status, 0) + self.assertIsNone(reason) + + def test_server_storage_fail_only_fails_the_job(self): + path = write_bep([ + result_event(SS, "FAILED"), + result_event(SS2, "PASSED"), + result_event(CD, "PASSED"), + result_event(CD2, "PASSED"), + ]) + try: + results, status, reason = ctr.classify(path, 3, TIERS, "server-storage") + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "failure", "cmd-dolt": "success"}) + self.assertEqual(status, 1) + self.assertIsNotNone(reason) + + def test_cmd_dolt_fail_only_leaves_the_job_green(self): + path = write_bep([ + result_event(SS, "PASSED"), + result_event(SS2, "PASSED"), + result_event(CD, "FAILED"), + result_event(CD2, "PASSED"), + ]) + try: + results, status, reason = ctr.classify(path, 3, TIERS, "server-storage") + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "success", "cmd-dolt": "failure"}) + # A cmd-dolt-only test failure must not fail the job: it stays + # advisory, exactly as bazel-cmd-dolt's own continue-on-error outside + # mode remote. + self.assertEqual(status, 0) + self.assertIsNone(reason) + + def test_both_tiers_fail(self): + path = write_bep([ + result_event(SS, "FAILED"), + result_event(SS2, "PASSED"), + result_event(CD, "TIMEOUT"), + result_event(CD2, "PASSED"), + ]) + try: + results, status, reason = ctr.classify(path, 3, TIERS, "server-storage") + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "failure", "cmd-dolt": "failure"}) + self.assertEqual(status, 1) + + def test_build_failure_fails_both_tiers_and_the_job(self): + # rc=1 (BUILD_FAILURE): not attributable to one tier, even though the + # BEP happens to show every known label passing (e.g. an unrelated + # target failed to build under --keep_going). + path = all_passed_bep() + try: + results, status, reason = ctr.classify(path, 1, TIERS, "server-storage") + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "failure", "cmd-dolt": "failure"}) + self.assertEqual(status, 1) + + def test_missing_rc_fails_both_tiers_and_the_job(self): + # should-fix 2 (bee-ghosttrack's review): a timed-out or cancelled + # `bazel test` step never writes $TEST_RC, so bazel.yml passes + # --rc "" rather than omitting the flag; this must take the "not 0 + # or 3" branch, not crash. + path = all_passed_bep() + try: + results, status, reason = ctr.classify(path, "", TIERS, "server-storage") + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "failure", "cmd-dolt": "failure"}) + self.assertEqual(status, 1) + self.assertIn("missing or not an integer", reason) + + def test_non_integer_rc_fails_both_tiers_and_the_job(self): + path = all_passed_bep() + try: + results, status, reason = ctr.classify(path, "abc", TIERS, "server-storage") + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "failure", "cmd-dolt": "failure"}) + self.assertEqual(status, 1) + self.assertIn("missing or not an integer", reason) + + def test_gate_tier_selects_which_tier_fails_the_job(self): + # Flip which tier gates: a cmd-dolt failure now fails the job, a + # server-storage failure does not. (Not bazel-dolt-int's own wiring, + # which always gates on server-storage; this just proves the script + # is not hardcoded to one tier name.) + path = write_bep([ + result_event(SS, "FAILED"), + result_event(SS2, "PASSED"), + result_event(CD, "PASSED"), + result_event(CD2, "PASSED"), + ]) + try: + results, status, reason = ctr.classify(path, 3, TIERS, "cmd-dolt") + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "failure", "cmd-dolt": "success"}) + self.assertEqual(status, 0) + + def test_unknown_failing_label_fails_every_tier_and_the_job(self): + # must-fix 1: a target tagged dolt-server-integration/dolt-server-cmd + # added without updating the --tier lists (bazel.yml's classify step + # args, or the --tier arg dict built here) must not read as success + # just because no --tier list claims its label. + path = all_passed_bep(extra=[result_event(UNKNOWN, "FAILED")]) + try: + results, status, reason = ctr.classify(path, 3, TIERS, "server-storage") + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "failure", "cmd-dolt": "failure"}) + self.assertEqual(status, 1) + self.assertIn(UNKNOWN, reason) + + def test_unknown_passing_label_does_not_affect_classification(self): + # A target outside every --tier that passed is not this script's + # business (it is not one of the two tiers bazel-dolt-int reports); + # only a non-PASSED result outside the tiers triggers the fail-closed + # path. + path = all_passed_bep(extra=[result_event(UNKNOWN, "PASSED")]) + try: + results, status, reason = ctr.classify(path, 0, TIERS, "server-storage") + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "success", "cmd-dolt": "success"}) + self.assertEqual(status, 0) + + def test_check_failure_forces_that_tier_failed_and_gates_the_job(self): + # should-fix 2: a per-tier check_testcases.py step's own outcome + # (bazel.yml's split "Every target and shard ran tests ()" + # steps) folds into that tier's result even though the BEP alone + # would call it a success (e.g. a zero-testcase shard). + path = all_passed_bep() + try: + results, status, reason = ctr.classify( + path, 0, TIERS, "server-storage", checks={"server-storage": "failure"} + ) + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "failure", "cmd-dolt": "success"}) + self.assertEqual(status, 1) + + def test_check_failure_on_non_gate_tier_leaves_the_job_green(self): + path = all_passed_bep() + try: + results, status, reason = ctr.classify( + path, 0, TIERS, "server-storage", checks={"cmd-dolt": "failure"} + ) + finally: + os.unlink(path) + self.assertEqual(results, {"server-storage": "success", "cmd-dolt": "failure"}) + self.assertEqual(status, 0) + + def test_check_cancelled_also_forces_failure(self): + path = all_passed_bep() + try: + results, _, _ = ctr.classify( + path, 0, TIERS, "server-storage", checks={"cmd-dolt": "cancelled"} + ) + finally: + os.unlink(path) + self.assertEqual(results["cmd-dolt"], "failure") + + def test_check_success_leaves_bep_result_alone(self): + path = write_bep([ + result_event(SS, "FAILED"), + result_event(SS2, "PASSED"), + result_event(CD, "PASSED"), + result_event(CD2, "PASSED"), + ]) + try: + results, status, _ = ctr.classify( + path, 3, TIERS, "server-storage", checks={"server-storage": "success"} + ) + finally: + os.unlink(path) + self.assertEqual(results["server-storage"], "failure") + self.assertEqual(status, 1) + + +class MainTest(unittest.TestCase): + def run_main(self, bep, rc, gate_tier="server-storage", extra_args=()): + out = tempfile.NamedTemporaryFile(mode="w", delete=False) + out.close() + try: + status = ctr.main([ + "--bep", bep, + "--rc", str(rc), + "--tier", f"server-storage={SS},{SS2}", + "--tier", f"cmd-dolt={CD},{CD2}", + "--gate-tier", gate_tier, + "--github-output", out.name, + *extra_args, + ]) + with open(out.name, encoding="utf-8") as f: + lines = set(f.read().splitlines()) + finally: + os.unlink(out.name) + return status, lines + + def test_writes_github_output_and_exits_1_on_gate_tier_failure(self): + bep = write_bep([ + result_event(SS, "FAILED"), + result_event(SS2, "PASSED"), + result_event(CD, "PASSED"), + result_event(CD2, "PASSED"), + ]) + try: + status, lines = self.run_main(bep, 3) + finally: + os.unlink(bep) + self.assertEqual(status, 1) + self.assertEqual(lines, {"server_storage_result=failure", "cmd_dolt_result=success"}) + + def test_empty_rc_exits_1_with_outputs_written_not_an_argparse_crash(self): + # should-fix 2: bazel.yml always passes --rc "$TEST_RC", so a timed + # out or cancelled test step reaches main() with --rc "" rather than + # omitting the flag; this must write both failure outputs and exit + # 1, not crash argparse with nothing written at all. + bep = all_passed_bep() + try: + status, lines = self.run_main(bep, "") + finally: + os.unlink(bep) + self.assertEqual(status, 1) + self.assertEqual(lines, {"server_storage_result=failure", "cmd_dolt_result=failure"}) + + def test_exits_0_when_only_the_non_gate_tier_fails(self): + bep = write_bep([ + result_event(SS, "PASSED"), + result_event(SS2, "PASSED"), + result_event(CD, "FAILED"), + result_event(CD2, "PASSED"), + ]) + try: + status, _ = self.run_main(bep, 3) + finally: + os.unlink(bep) + self.assertEqual(status, 0) + + def test_unknown_failing_label_exits_1_through_main(self): + bep = all_passed_bep(extra=[result_event(UNKNOWN, "FAILED")]) + try: + status, lines = self.run_main(bep, 3) + finally: + os.unlink(bep) + self.assertEqual(status, 1) + self.assertEqual(lines, {"server_storage_result=failure", "cmd_dolt_result=failure"}) + + def test_check_flag_folds_into_tier_output(self): + bep = all_passed_bep() + try: + status, lines = self.run_main(bep, 0, extra_args=["--check", "cmd-dolt=failure"]) + finally: + os.unlink(bep) + self.assertEqual(status, 0) + self.assertEqual(lines, {"server_storage_result=success", "cmd_dolt_result=failure"}) + + def test_repeated_tier_name_is_rejected(self): + bep = all_passed_bep() + out = tempfile.NamedTemporaryFile(mode="w", delete=False) + out.close() + try: + with self.assertRaises(SystemExit): + ctr.main([ + "--bep", bep, + "--rc", "0", + "--tier", f"server-storage={SS},{SS2}", + "--tier", f"cmd-dolt={CD},{CD2}", + "--tier", "cmd-dolt=//other:t", + "--gate-tier", "server-storage", + "--github-output", out.name, + ]) + finally: + os.unlink(bep) + os.unlink(out.name) + + def test_repeated_gate_tier_is_rejected(self): + bep = all_passed_bep() + out = tempfile.NamedTemporaryFile(mode="w", delete=False) + out.close() + try: + with self.assertRaises(SystemExit): + ctr.main([ + "--bep", bep, + "--rc", "0", + "--tier", f"server-storage={SS},{SS2}", + "--tier", f"cmd-dolt={CD},{CD2}", + "--gate-tier", "server-storage", + "--gate-tier", "cmd-dolt", + "--github-output", out.name, + ]) + finally: + os.unlink(bep) + os.unlink(out.name) + + def test_repeated_check_name_is_rejected(self): + bep = all_passed_bep() + out = tempfile.NamedTemporaryFile(mode="w", delete=False) + out.close() + try: + with self.assertRaises(SystemExit): + ctr.main([ + "--bep", bep, + "--rc", "0", + "--tier", f"server-storage={SS},{SS2}", + "--tier", f"cmd-dolt={CD},{CD2}", + "--gate-tier", "server-storage", + "--github-output", out.name, + "--check", "cmd-dolt=failure", + "--check", "cmd-dolt=success", + ]) + finally: + os.unlink(bep) + os.unlink(out.name) + + def test_unknown_flag_is_rejected(self): + bep = all_passed_bep() + out = tempfile.NamedTemporaryFile(mode="w", delete=False) + out.close() + try: + with self.assertRaises(SystemExit): + ctr.main([ + "--bep", bep, + "--rc", "0", + "--tier", f"server-storage={SS},{SS2}", + "--gate-tier", "server-storage", + "--github-output", out.name, + "--gate-tier-extra", "server-storage", + ]) + finally: + os.unlink(bep) + os.unlink(out.name) + + def test_check_for_unknown_tier_name_is_rejected(self): + bep = all_passed_bep() + out = tempfile.NamedTemporaryFile(mode="w", delete=False) + out.close() + try: + with self.assertRaises(SystemExit): + ctr.main([ + "--bep", bep, + "--rc", "0", + "--tier", f"server-storage={SS},{SS2}", + "--gate-tier", "server-storage", + "--github-output", out.name, + "--check", "not-a-tier=failure", + ]) + finally: + os.unlink(bep) + os.unlink(out.name) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/bazel/shard_budget.py b/tools/bazel/shard_budget.py index 22dd10506..6416324a8 100644 --- a/tools/bazel/shard_budget.py +++ b/tools/bazel/shard_budget.py @@ -32,6 +32,26 @@ Day one: this prints `::warning` and always exits 0 (CI must not block on it yet). Pass --fail-on-imbalance (a workflow step can gate this behind a repository variable) to exit 1 and print `::error` instead, once the budget has run clean for a while. + +--label (repeatable) restricts which sharded targets this invocation judges, +by exact label. A union job's BEP has every member's shards in it; pass each +label whose owning lane's job this script runs in (item 5 A2: +bazel-dolt-int's step passes only server-storage's +//internal/storage/dolt:dolt_server_full_test, so it keeps judging that tier +alone and not cmd-dolt's bd_dolt_server_test / regression_test shards, which +no job budgeted before the union and which this script has never judged). +Omitted (bazel-server-storage's own step, unchanged): every sharded target in +the BEP, as before. + +A --label naming a target with no testResult event in the BEP at all (cached +or executed) is a workflow authoring error (a typo, or a target renamed out +from under a pinned step), not a budget outcome: the step prints ::error and +exits 1 rather than silently reporting "no sharded test target executed two +or more shards" (PR #7535 review, should-fix 3, carried across both rounds). +A label that is in the BEP but had every shard served from cache this run -- +so it has zero executed shards, same as a label with only one -- is not an +error: a fully cache-hit invocation is a legitimate outcome evaluate() +already reports as "nothing to judge" for that target, same as before. """ import argparse @@ -79,6 +99,35 @@ def read_shard_durations(path): return shards +def read_tested_labels(path): + """Return the set of every label with at least one testResult event in + the BEP, executed or cached. + + Separate from read_shard_durations, which only counts executed (non- + cached) attempts: a target whose every shard was a cache hit this + invocation has no entry in read_shard_durations at all, which is a + legitimate "nothing to judge" outcome (evaluate() already skips it), + not a --label typo. This set is what --label's fail-closed check + (main()) tests against instead, so a fully-cached run does not falsely + report an unknown label. + """ + labels = set() + with open(path, encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + try: + ev = json.loads(line) + except ValueError: + continue + tr_id = ev.get("id", {}).get("testResult") + tr = ev.get("testResult") + if not tr_id or not tr: + continue + labels.add(tr_id["label"]) + return labels + + class Imbalance: """One sharded test target's balance verdict.""" @@ -113,11 +162,19 @@ class Imbalance: ) -def evaluate(shard_durations): +def evaluate(shard_durations, labels=None): """Return a sorted list of Imbalance for every target with >= 2 executed - shards this invocation, whether or not it crosses the warn threshold.""" + shards this invocation, whether or not it crosses the warn threshold. + + labels, if given (a set), restricts evaluation to those exact labels: a + union job's BEP covers every member's shards, and a caller that only + owns one member's shard budget passes its labels so the other member's + shards are never judged here. + """ out = [] for label, per_shard in sorted(shard_durations.items()): + if labels is not None and label not in labels: + continue if len(per_shard) < 2: continue durations = list(per_shard.values()) @@ -137,12 +194,25 @@ def main(argv=None): action="store_true", help="exit 1 (and print ::error) past the fail threshold; default is warn-only", ) + ap.add_argument( + "--label", + action="append", + default=None, + help="restrict to this exact target label (repeatable); default: every sharded target in the BEP", + ) args = ap.parse_args(argv) shard_durations = read_shard_durations(args.bep) - results = evaluate(shard_durations) failed = False + if args.label: + tested = read_tested_labels(args.bep) + for label in sorted(set(args.label) - tested): + failed = True + print(f"::error title=shard budget::{label}: --label, but the BEP has no result for it") + + results = evaluate(shard_durations, labels=set(args.label) if args.label else None) + for r in results: if r.fail and args.fail_on_imbalance: failed = True diff --git a/tools/bazel/shard_budget_test.py b/tools/bazel/shard_budget_test.py index 0c0718bcf..4f1ccb91d 100644 --- a/tools/bazel/shard_budget_test.py +++ b/tools/bazel/shard_budget_test.py @@ -140,6 +140,24 @@ class EvaluateTest(unittest.TestCase): }) self.assertEqual([r.label for r in results], ["//b:t", "//a:t"]) + def test_labels_restricts_to_exact_matches(self): + # item 5 A2's union BEP has both tiers' shards; a lane that owns + # only one target's budget must not be skewed by the other's. + shards = { + "//a:t": {1: 10_000, 2: 20_000}, + "//b:t": {1: 80_000, 2: 900_000}, + } + results = shard_budget.evaluate(shards, labels={"//a:t"}) + self.assertEqual([r.label for r in results], ["//a:t"]) + + def test_labels_none_judges_every_sharded_target(self): + shards = { + "//a:t": {1: 10_000, 2: 20_000}, + "//b:t": {1: 80_000, 2: 900_000}, + } + results = shard_budget.evaluate(shards, labels=None) + self.assertEqual({r.label for r in results}, {"//a:t", "//b:t"}) + class MainTest(unittest.TestCase): def test_warn_only_exits_zero(self): @@ -175,6 +193,67 @@ class MainTest(unittest.TestCase): os.unlink(path) self.assertEqual(rc, 0) + def test_label_scopes_to_exact_target_ignoring_others_imbalance(self): + # item 5 A2's bazel-dolt-int Shard balance step: --label scopes to + # server-storage's own target, so a wildly imbalanced cmd-dolt shard + # elsewhere in the same union BEP does not fail this invocation. + storage_shards = {i: 80_000 for i in range(1, 10)} + cmd_shards = {1: 10_000, 2: 900_000} # would fail on its own + events = [test_result_event("//storage:t", i, ms) for i, ms in storage_shards.items()] + events += [test_result_event("//cmd:t", i, ms) for i, ms in cmd_shards.items()] + path = write_bep(events) + try: + rc = shard_budget.main(["--bep", path, "--fail-on-imbalance", "--label", "//storage:t"]) + finally: + os.unlink(path) + self.assertEqual(rc, 0) + + def test_unknown_label_fails_even_without_fail_on_imbalance(self): + # PR #7535 review, should-fix 3 (carried): a --label naming a target + # with no testResult event in the BEP at all is a workflow authoring + # error (a typo, or a renamed target), not a quiet "nothing to + # judge" success. + events = [test_result_event("//storage:t", i, 80_000) for i in range(1, 10)] + path = write_bep(events) + try: + rc = shard_budget.main(["--bep", path, "--label", "//nope:x"]) + finally: + os.unlink(path) + self.assertEqual(rc, 1) + + def test_label_present_with_one_shard_is_not_an_error(self): + # A label can legitimately have zero or one executed shard this run + # (the rest were cache hits): evaluate() already skips it as + # insufficient evidence, and that must stay a 0, not the unknown- + # label error above. + events = [test_result_event("//storage:t", 1, 80_000)] + path = write_bep(events) + try: + rc = shard_budget.main(["--bep", path, "--label", "//storage:t"]) + finally: + os.unlink(path) + self.assertEqual(rc, 0) + + def test_label_with_every_shard_cached_is_not_an_error(self): + # A fully cache-hit invocation of a sharded target is a legitimate + # outcome (nothing executed, so nothing to judge for balance), not a + # --label typo: read_shard_durations has zero entries for it, but + # read_tested_labels still sees it via its cached testResult events. + # This was a real CI regression (PR #7535's own dolt-int run went + # red here on a fully-cached //internal/storage/dolt: + # dolt_server_full_test) that the should-fix 3 fix above introduced + # and this test pins against recurring. + events = [ + test_result_event("//storage:t", i, 80_000, cached=True) + for i in range(1, 10) + ] + path = write_bep(events) + try: + rc = shard_budget.main(["--bep", path, "--label", "//storage:t"]) + finally: + os.unlink(path) + self.assertEqual(rc, 0) + if __name__ == "__main__": unittest.main()