From 085e8a8abaf39cff369d10e8ac91dada4e45ccbc Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 03:33:13 +0000 Subject: [PATCH] ci: warn-only shard-balance budget for bazel-server-storage rbe-ci-cost-latency-study.md recommendation 4: this tier's last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. tools/bazel/shard_budget.py reads the job's BEP file and prints ::warning when a sharded target's slowest shard exceeds max(150s, 2x median); --fail-on-imbalance (not wired in yet) switches that to ::error and a nonzero exit once the budget has run clean for a while. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/bazel.yml | 9 ++ scripts/ci_workflow_test.go | 25 ++++- tools/bazel/shard_budget.py | 162 ++++++++++++++++++++++++++++ tools/bazel/shard_budget_test.py | 180 +++++++++++++++++++++++++++++++ 4 files changed, 371 insertions(+), 5 deletions(-) create mode 100644 tools/bazel/shard_budget.py create mode 100644 tools/bazel/shard_budget_test.py diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index f0da60d84..fc758e7f5 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1707,6 +1707,15 @@ jobs: --suite //internal/storage/dolt:dolt_server_full_test .github/scripts/server-storage-test-shard.sh 16 --whole //internal/storage/dolt:dolt_server_conformance_test + # rbe-ci-cost-latency-study.md recommendation 4: this tier's last + # shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. Warns + # (::warning) when the slowest shard this run is far above the + # median; does not fail the job yet (--fail-on-imbalance switches + # that on once the budget has run clean for a while). + - name: Shard balance + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: python3 tools/bazel/shard_budget.py --bep "$RUNNER_TEMP/bazel-bep.json" + - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index e8ce12ca1..0cbc8d3b5 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -5030,23 +5030,38 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { // require) or of bazel-integration (which a caller may switch off, // while the server tier always runs remotely). workflow := readCIWorkflow(t, bazelWorkflowName) - for _, c := range []struct{ job, config, logs string }{ - {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs"}, - {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs"}, + for _, c := range []struct { + job, config, logs string + wantSteps int + }{ + {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 8}, + // One more than bazel-proxied: a "Shard balance" step (rbe-ci-cost- + // latency-study.md recommendation 4), since this is the tier whose + // last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. + {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 9}, } { job := workflow.job(t, c.job) if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 { t.Errorf("%s timeout-minutes = %d; it runs remotely only (longest shard ~2-8 min), keep it at most 30", c.job, job.TimeoutMinutes) } assertBazelTierStep(t, job, c.job, c.config) - if n := len(job.Steps); n != 8 { - t.Errorf("%s has %d steps, want checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, log upload, result recorder", c.job, n) + if n := len(job.Steps); n != c.wantSteps { + t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, result recorder)", c.job, n, c.wantSteps) } logs := job.step(t, "Upload test logs") if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { t.Errorf("%s test-log upload: if=%q name=%q uses=%q", c.job, logs.If, logs.With["name"], logs.Uses) } } + if serverStorage := workflow.job(t, bazelServerJobName); true { + budget := serverStorage.step(t, "Shard balance") + if budget.If != "${{ always() && steps.test.outcome != 'skipped' }}" { + t.Errorf("%s Shard balance step if=%q", bazelServerJobName, budget.If) + } + if want := `python3 tools/bazel/shard_budget.py --bep "$RUNNER_TEMP/bazel-bep.json"`; strings.TrimSpace(budget.Run) != want { + t.Errorf("%s Shard balance step runs %q, want %q", bazelServerJobName, strings.TrimSpace(budget.Run), want) + } + } for name, j := range workflow.Jobs { for _, step := range j.Steps { proxied := strings.Contains(step.Run, "--config=doltserver-proxied") diff --git a/tools/bazel/shard_budget.py b/tools/bazel/shard_budget.py new file mode 100644 index 000000000..22dd10506 --- /dev/null +++ b/tools/bazel/shard_budget.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Warn (or fail) when a sharded test's slowest shard is far above its median. + +Usage: shard_budget.py --bep [--fail-on-imbalance] + +Rationale (rbe-ci-cost-latency-study.md, recommendation 4): an unbalanced +shard split quietly grows the critical path of whichever lane runs it. gascity +#7233/#7241 showed the gain of re-splitting a hot shard; this script keeps +that gain from regressing and catches the tier this repository has today: +//internal/storage/dolt:dolt_server_full_test's last shard has trailed the +rest by 4.3-4.5 min in 2 of 8 runs (study Section 3.4). + +Reads one `bazel test` invocation's --build_event_json_file (the same file +tools/bazel/check_testcases.py and check_shard_coverage.py already read) and, +for every sharded test target (shard_count > 1) that this invocation actually +executed at least two shards of, computes: + + median = the median wall time of the shards that executed (not a cache + hit; a cached shard's duration says nothing about balance) + slowest = the maximum of those + +A shard with no executed result (every attempt was a cache hit) is left out +of both: it ran nowhere this invocation. A target with fewer than two shards +left is skipped (not enough evidence this run). + +Thresholds (study Section 6.4, picked from its shard data): + + warn when slowest > max(150s, 2 x median) + fail when slowest > max(300s, 3 x median) -- only with --fail-on-imbalance + +Day one: this prints `::warning` and always exits 0 (CI must not block on +it yet). Pass --fail-on-imbalance (a workflow step can gate this behind a +repository variable) to exit 1 and print `::error` instead, once the budget +has run clean for a while. +""" + +import argparse +import json +import statistics +import sys + +WARN_FLOOR_S = 150.0 +WARN_FACTOR = 2.0 +FAIL_FLOOR_S = 300.0 +FAIL_FACTOR = 3.0 + + +def read_shard_durations(path): + """Return {label: {shard_index: executed_duration_ms}} from a BEP file. + + Only testResult events are read (as equivalence.py's read_bep does); a + shard's duration is the longest executed (non-cached) attempt reported + for it. shard defaults to 1 (an unsharded test), matching Bazel's BEP, + which omits the field for shard_count == 1. + """ + shards = {} + with open(path, encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + try: + ev = json.loads(line) + except ValueError: + continue + tr_id = ev.get("id", {}).get("testResult") + tr = ev.get("testResult") + if not tr_id or not tr: + continue + cached = bool(tr.get("cachedLocally")) or bool( + (tr.get("executionInfo") or {}).get("cachedRemotely") + ) + if cached: + continue + label = tr_id["label"] + shard = int(tr_id.get("shard") or 1) + ms = int(tr.get("testAttemptDurationMillis") or 0) + per_label = shards.setdefault(label, {}) + per_label[shard] = max(per_label.get(shard, 0), ms) + return shards + + +class Imbalance: + """One sharded test target's balance verdict.""" + + def __init__(self, label, shards, median_ms, slowest_ms, slowest_shard): + self.label = label + self.shards = shards + self.median_ms = median_ms + self.slowest_ms = slowest_ms + self.slowest_shard = slowest_shard + + @property + def warn_threshold_ms(self): + return max(WARN_FLOOR_S * 1000, WARN_FACTOR * self.median_ms) + + @property + def fail_threshold_ms(self): + return max(FAIL_FLOOR_S * 1000, FAIL_FACTOR * self.median_ms) + + @property + def warn(self): + return self.slowest_ms > self.warn_threshold_ms + + @property + def fail(self): + return self.slowest_ms > self.fail_threshold_ms + + def message(self): + return ( + f"{self.label}: shard {self.slowest_shard} of {self.shards} took " + f"{self.slowest_ms / 1000:.0f}s against a median of {self.median_ms / 1000:.0f}s " + f"(warn over {self.warn_threshold_ms / 1000:.0f}s, fail over {self.fail_threshold_ms / 1000:.0f}s)" + ) + + +def evaluate(shard_durations): + """Return a sorted list of Imbalance for every target with >= 2 executed + shards this invocation, whether or not it crosses the warn threshold.""" + out = [] + for label, per_shard in sorted(shard_durations.items()): + if len(per_shard) < 2: + continue + durations = list(per_shard.values()) + median_ms = statistics.median(durations) + slowest_ms = max(durations) + slowest_shard = max(per_shard, key=per_shard.get) + out.append(Imbalance(label, len(per_shard), median_ms, slowest_ms, slowest_shard)) + out.sort(key=lambda i: i.slowest_ms, reverse=True) + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") + ap.add_argument( + "--fail-on-imbalance", + action="store_true", + help="exit 1 (and print ::error) past the fail threshold; default is warn-only", + ) + args = ap.parse_args(argv) + + shard_durations = read_shard_durations(args.bep) + results = evaluate(shard_durations) + + failed = False + for r in results: + if r.fail and args.fail_on_imbalance: + failed = True + print(f"::error title=shard budget::{r.message()}") + elif r.warn: + print(f"::warning title=shard budget::{r.message()}") + else: + print(f"ok: {r.message()}") + + if not results: + print("shard budget: no sharded test target executed two or more shards in this invocation") + + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/shard_budget_test.py b/tools/bazel/shard_budget_test.py new file mode 100644 index 000000000..0c0718bcf --- /dev/null +++ b/tools/bazel/shard_budget_test.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Unit tests for shard_budget.py. + +Run directly: python3 tools/bazel/shard_budget_test.py +(or: python3 -m unittest tools.bazel.shard_budget_test, from the repo root) +""" + +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import shard_budget # noqa: E402 + + +def write_bep(events): + f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8") + for ev in events: + f.write(json.dumps(ev) + "\n") + f.close() + return f.name + + +def test_result_event(label, shard, ms, cached=False, cached_remotely=False): + return { + "id": {"testResult": {"label": label, "shard": shard}}, + "testResult": { + "testAttemptDurationMillis": str(ms), + "cachedLocally": cached, + "executionInfo": {"cachedRemotely": cached_remotely, "strategy": "remote"}, + }, + } + + +class ReadShardDurationsTest(unittest.TestCase): + def test_collects_executed_durations_per_shard(self): + path = write_bep([ + test_result_event("//x:t", 1, 10_000), + test_result_event("//x:t", 2, 20_000), + ]) + try: + got = shard_budget.read_shard_durations(path) + finally: + os.unlink(path) + self.assertEqual(got, {"//x:t": {1: 10_000, 2: 20_000}}) + + def test_excludes_cached_results(self): + path = write_bep([ + test_result_event("//x:t", 1, 10_000), + test_result_event("//x:t", 2, 999_000, cached_remotely=True), + ]) + try: + got = shard_budget.read_shard_durations(path) + finally: + os.unlink(path) + # Shard 2 was a cache hit: it contributes no duration. + self.assertEqual(got, {"//x:t": {1: 10_000}}) + + def test_unsharded_defaults_to_shard_1(self): + path = write_bep([test_result_event("//x:t", None, 5_000)]) + try: + got = shard_budget.read_shard_durations(path) + finally: + os.unlink(path) + self.assertEqual(got, {"//x:t": {1: 5_000}}) + + def test_ignores_non_test_result_lines(self): + path = write_bep([{"id": {"progress": {}}, "progress": {}}]) + try: + got = shard_budget.read_shard_durations(path) + finally: + os.unlink(path) + self.assertEqual(got, {}) + + def test_malformed_line_is_skipped(self): + f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8") + f.write("not json\n") + f.write(json.dumps(test_result_event("//x:t", 1, 1_000)) + "\n") + f.close() + try: + got = shard_budget.read_shard_durations(f.name) + finally: + os.unlink(f.name) + self.assertEqual(got, {"//x:t": {1: 1_000}}) + + +class EvaluateTest(unittest.TestCase): + def test_balanced_shards_do_not_warn(self): + # gascity #7241-shaped data: every shard within a few seconds of the + # others never crosses max(150s, 2x median). + results = shard_budget.evaluate({"//x:t": {i: 60_000 + i * 1000 for i in range(1, 11)}}) + self.assertEqual(len(results), 1) + r = results[0] + self.assertFalse(r.warn) + self.assertFalse(r.fail) + + def test_single_long_pole_warns(self): + # gascity acceptance shard 4 shape: one shard far above the rest. + shards = {i: 80_000 for i in range(1, 10)} + shards[4] = 725_000 + results = shard_budget.evaluate({"//acceptance:t": shards}) + r = results[0] + self.assertTrue(r.warn) + self.assertEqual(r.slowest_shard, 4) + self.assertAlmostEqual(r.slowest_ms, 725_000) + + def test_fail_threshold_is_stricter_than_warn(self): + # beads server-Dolt storage tier shape: a straggler 4.3-4.5 min + # behind an otherwise ~80s median. Over the warn floor (150s, 2x + # median) but under the fail floor (300s, 3x median) until it grows + # further. + shards = {i: 80_000 for i in range(1, 16)} + shards[16] = 250_000 + r = shard_budget.evaluate({"//storage/dolt:t": shards})[0] + self.assertTrue(r.warn) + self.assertFalse(r.fail) + + shards[16] = 400_000 + r = shard_budget.evaluate({"//storage/dolt:t": shards})[0] + self.assertTrue(r.warn) + self.assertTrue(r.fail) + + def test_floor_applies_below_tiny_medians(self): + # A fast, lopsided split (two 5s shards, one 200s shard) must still + # warn even though 2x the 5s median is nowhere near the slowest + # shard; the 150s floor catches it, not the ratio. + r = shard_budget.evaluate({"//x:t": {1: 5_000, 2: 5_000, 3: 200_000}})[0] + self.assertTrue(r.warn) + + def test_fewer_than_two_shards_is_skipped(self): + self.assertEqual(shard_budget.evaluate({"//x:t": {1: 10_000}}), []) + self.assertEqual(shard_budget.evaluate({"//x:t": {}}), []) + + def test_sorted_slowest_first(self): + results = shard_budget.evaluate({ + "//a:t": {1: 10_000, 2: 20_000}, + "//b:t": {1: 10_000, 2: 500_000}, + }) + self.assertEqual([r.label for r in results], ["//b:t", "//a:t"]) + + +class MainTest(unittest.TestCase): + def test_warn_only_exits_zero(self): + shards = {i: 80_000 for i in range(1, 10)} + shards[4] = 900_000 + events = [test_result_event("//x:t", i, ms) for i, ms in shards.items()] + path = write_bep(events) + try: + rc = shard_budget.main(["--bep", path]) + finally: + os.unlink(path) + self.assertEqual(rc, 0) + + def test_fail_on_imbalance_exits_nonzero_past_fail_threshold(self): + shards = {i: 80_000 for i in range(1, 10)} + shards[4] = 900_000 + events = [test_result_event("//x:t", i, ms) for i, ms in shards.items()] + path = write_bep(events) + try: + rc = shard_budget.main(["--bep", path, "--fail-on-imbalance"]) + finally: + os.unlink(path) + self.assertEqual(rc, 1) + + def test_fail_on_imbalance_still_zero_under_fail_threshold(self): + shards = {i: 80_000 for i in range(1, 10)} + shards[4] = 200_000 + events = [test_result_event("//x:t", i, ms) for i, ms in shards.items()] + path = write_bep(events) + try: + rc = shard_budget.main(["--bep", path, "--fail-on-imbalance"]) + finally: + os.unlink(path) + self.assertEqual(rc, 0) + + +if __name__ == "__main__": + unittest.main()