ci: warn-only shard-balance budget for bazel-server-storage
rbe-ci-cost-latency-study.md recommendation 4: this tier's last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. tools/bazel/shard_budget.py reads the job's BEP file and prints ::warning when a sharded target's slowest shard exceeds max(150s, 2x median); --fail-on-imbalance (not wired in yet) switches that to ::error and a nonzero exit once the budget has run clean for a while. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
5e73670866
commit
085e8a8aba
4 changed files with 371 additions and 5 deletions
9
.github/workflows/bazel.yml
vendored
9
.github/workflows/bazel.yml
vendored
|
|
@ -1707,6 +1707,15 @@ jobs:
|
|||
--suite //internal/storage/dolt:dolt_server_full_test .github/scripts/server-storage-test-shard.sh 16
|
||||
--whole //internal/storage/dolt:dolt_server_conformance_test
|
||||
|
||||
# rbe-ci-cost-latency-study.md recommendation 4: this tier's last
|
||||
# shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. Warns
|
||||
# (::warning) when the slowest shard this run is far above the
|
||||
# median; does not fail the job yet (--fail-on-imbalance switches
|
||||
# that on once the budget has run clean for a while).
|
||||
- name: Shard balance
|
||||
if: ${{ always() && steps.test.outcome != 'skipped' }}
|
||||
run: python3 tools/bazel/shard_budget.py --bep "$RUNNER_TEMP/bazel-bep.json"
|
||||
|
||||
- name: Upload test logs
|
||||
if: ${{ failure() && steps.test.outcome != 'skipped' }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
|
|
|
|||
|
|
@ -5030,23 +5030,38 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) {
|
|||
// require) or of bazel-integration (which a caller may switch off,
|
||||
// while the server tier always runs remotely).
|
||||
workflow := readCIWorkflow(t, bazelWorkflowName)
|
||||
for _, c := range []struct{ job, config, logs string }{
|
||||
{bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs"},
|
||||
{bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs"},
|
||||
for _, c := range []struct {
|
||||
job, config, logs string
|
||||
wantSteps int
|
||||
}{
|
||||
{bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 8},
|
||||
// One more than bazel-proxied: a "Shard balance" step (rbe-ci-cost-
|
||||
// latency-study.md recommendation 4), since this is the tier whose
|
||||
// last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs.
|
||||
{bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 9},
|
||||
} {
|
||||
job := workflow.job(t, c.job)
|
||||
if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 {
|
||||
t.Errorf("%s timeout-minutes = %d; it runs remotely only (longest shard ~2-8 min), keep it at most 30", c.job, job.TimeoutMinutes)
|
||||
}
|
||||
assertBazelTierStep(t, job, c.job, c.config)
|
||||
if n := len(job.Steps); n != 8 {
|
||||
t.Errorf("%s has %d steps, want checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, log upload, result recorder", c.job, n)
|
||||
if n := len(job.Steps); n != c.wantSteps {
|
||||
t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, result recorder)", c.job, n, c.wantSteps)
|
||||
}
|
||||
logs := job.step(t, "Upload test logs")
|
||||
if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") {
|
||||
t.Errorf("%s test-log upload: if=%q name=%q uses=%q", c.job, logs.If, logs.With["name"], logs.Uses)
|
||||
}
|
||||
}
|
||||
if serverStorage := workflow.job(t, bazelServerJobName); true {
|
||||
budget := serverStorage.step(t, "Shard balance")
|
||||
if budget.If != "${{ always() && steps.test.outcome != 'skipped' }}" {
|
||||
t.Errorf("%s Shard balance step if=%q", bazelServerJobName, budget.If)
|
||||
}
|
||||
if want := `python3 tools/bazel/shard_budget.py --bep "$RUNNER_TEMP/bazel-bep.json"`; strings.TrimSpace(budget.Run) != want {
|
||||
t.Errorf("%s Shard balance step runs %q, want %q", bazelServerJobName, strings.TrimSpace(budget.Run), want)
|
||||
}
|
||||
}
|
||||
for name, j := range workflow.Jobs {
|
||||
for _, step := range j.Steps {
|
||||
proxied := strings.Contains(step.Run, "--config=doltserver-proxied")
|
||||
|
|
|
|||
162
tools/bazel/shard_budget.py
Normal file
162
tools/bazel/shard_budget.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Warn (or fail) when a sharded test's slowest shard is far above its median.
|
||||
|
||||
Usage: shard_budget.py --bep <build_event_json_file> [--fail-on-imbalance]
|
||||
|
||||
Rationale (rbe-ci-cost-latency-study.md, recommendation 4): an unbalanced
|
||||
shard split quietly grows the critical path of whichever lane runs it. gascity
|
||||
#7233/#7241 showed the gain of re-splitting a hot shard; this script keeps
|
||||
that gain from regressing and catches the tier this repository has today:
|
||||
//internal/storage/dolt:dolt_server_full_test's last shard has trailed the
|
||||
rest by 4.3-4.5 min in 2 of 8 runs (study Section 3.4).
|
||||
|
||||
Reads one `bazel test` invocation's --build_event_json_file (the same file
|
||||
tools/bazel/check_testcases.py and check_shard_coverage.py already read) and,
|
||||
for every sharded test target (shard_count > 1) that this invocation actually
|
||||
executed at least two shards of, computes:
|
||||
|
||||
median = the median wall time of the shards that executed (not a cache
|
||||
hit; a cached shard's duration says nothing about balance)
|
||||
slowest = the maximum of those
|
||||
|
||||
A shard with no executed result (every attempt was a cache hit) is left out
|
||||
of both: it ran nowhere this invocation. A target with fewer than two shards
|
||||
left is skipped (not enough evidence this run).
|
||||
|
||||
Thresholds (study Section 6.4, picked from its shard data):
|
||||
|
||||
warn when slowest > max(150s, 2 x median)
|
||||
fail when slowest > max(300s, 3 x median) -- only with --fail-on-imbalance
|
||||
|
||||
Day one: this prints `::warning` and always exits 0 (CI must not block on
|
||||
it yet). Pass --fail-on-imbalance (a workflow step can gate this behind a
|
||||
repository variable) to exit 1 and print `::error` instead, once the budget
|
||||
has run clean for a while.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
|
||||
WARN_FLOOR_S = 150.0
|
||||
WARN_FACTOR = 2.0
|
||||
FAIL_FLOOR_S = 300.0
|
||||
FAIL_FACTOR = 3.0
|
||||
|
||||
|
||||
def read_shard_durations(path):
|
||||
"""Return {label: {shard_index: executed_duration_ms}} from a BEP file.
|
||||
|
||||
Only testResult events are read (as equivalence.py's read_bep does); a
|
||||
shard's duration is the longest executed (non-cached) attempt reported
|
||||
for it. shard defaults to 1 (an unsharded test), matching Bazel's BEP,
|
||||
which omits the field for shard_count == 1.
|
||||
"""
|
||||
shards = {}
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
tr_id = ev.get("id", {}).get("testResult")
|
||||
tr = ev.get("testResult")
|
||||
if not tr_id or not tr:
|
||||
continue
|
||||
cached = bool(tr.get("cachedLocally")) or bool(
|
||||
(tr.get("executionInfo") or {}).get("cachedRemotely")
|
||||
)
|
||||
if cached:
|
||||
continue
|
||||
label = tr_id["label"]
|
||||
shard = int(tr_id.get("shard") or 1)
|
||||
ms = int(tr.get("testAttemptDurationMillis") or 0)
|
||||
per_label = shards.setdefault(label, {})
|
||||
per_label[shard] = max(per_label.get(shard, 0), ms)
|
||||
return shards
|
||||
|
||||
|
||||
class Imbalance:
|
||||
"""One sharded test target's balance verdict."""
|
||||
|
||||
def __init__(self, label, shards, median_ms, slowest_ms, slowest_shard):
|
||||
self.label = label
|
||||
self.shards = shards
|
||||
self.median_ms = median_ms
|
||||
self.slowest_ms = slowest_ms
|
||||
self.slowest_shard = slowest_shard
|
||||
|
||||
@property
|
||||
def warn_threshold_ms(self):
|
||||
return max(WARN_FLOOR_S * 1000, WARN_FACTOR * self.median_ms)
|
||||
|
||||
@property
|
||||
def fail_threshold_ms(self):
|
||||
return max(FAIL_FLOOR_S * 1000, FAIL_FACTOR * self.median_ms)
|
||||
|
||||
@property
|
||||
def warn(self):
|
||||
return self.slowest_ms > self.warn_threshold_ms
|
||||
|
||||
@property
|
||||
def fail(self):
|
||||
return self.slowest_ms > self.fail_threshold_ms
|
||||
|
||||
def message(self):
|
||||
return (
|
||||
f"{self.label}: shard {self.slowest_shard} of {self.shards} took "
|
||||
f"{self.slowest_ms / 1000:.0f}s against a median of {self.median_ms / 1000:.0f}s "
|
||||
f"(warn over {self.warn_threshold_ms / 1000:.0f}s, fail over {self.fail_threshold_ms / 1000:.0f}s)"
|
||||
)
|
||||
|
||||
|
||||
def evaluate(shard_durations):
|
||||
"""Return a sorted list of Imbalance for every target with >= 2 executed
|
||||
shards this invocation, whether or not it crosses the warn threshold."""
|
||||
out = []
|
||||
for label, per_shard in sorted(shard_durations.items()):
|
||||
if len(per_shard) < 2:
|
||||
continue
|
||||
durations = list(per_shard.values())
|
||||
median_ms = statistics.median(durations)
|
||||
slowest_ms = max(durations)
|
||||
slowest_shard = max(per_shard, key=per_shard.get)
|
||||
out.append(Imbalance(label, len(per_shard), median_ms, slowest_ms, slowest_shard))
|
||||
out.sort(key=lambda i: i.slowest_ms, reverse=True)
|
||||
return out
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run")
|
||||
ap.add_argument(
|
||||
"--fail-on-imbalance",
|
||||
action="store_true",
|
||||
help="exit 1 (and print ::error) past the fail threshold; default is warn-only",
|
||||
)
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
shard_durations = read_shard_durations(args.bep)
|
||||
results = evaluate(shard_durations)
|
||||
|
||||
failed = False
|
||||
for r in results:
|
||||
if r.fail and args.fail_on_imbalance:
|
||||
failed = True
|
||||
print(f"::error title=shard budget::{r.message()}")
|
||||
elif r.warn:
|
||||
print(f"::warning title=shard budget::{r.message()}")
|
||||
else:
|
||||
print(f"ok: {r.message()}")
|
||||
|
||||
if not results:
|
||||
print("shard budget: no sharded test target executed two or more shards in this invocation")
|
||||
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
180
tools/bazel/shard_budget_test.py
Normal file
180
tools/bazel/shard_budget_test.py
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Unit tests for shard_budget.py.
|
||||
|
||||
Run directly: python3 tools/bazel/shard_budget_test.py
|
||||
(or: python3 -m unittest tools.bazel.shard_budget_test, from the repo root)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import shard_budget # noqa: E402
|
||||
|
||||
|
||||
def write_bep(events):
|
||||
f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8")
|
||||
for ev in events:
|
||||
f.write(json.dumps(ev) + "\n")
|
||||
f.close()
|
||||
return f.name
|
||||
|
||||
|
||||
def test_result_event(label, shard, ms, cached=False, cached_remotely=False):
|
||||
return {
|
||||
"id": {"testResult": {"label": label, "shard": shard}},
|
||||
"testResult": {
|
||||
"testAttemptDurationMillis": str(ms),
|
||||
"cachedLocally": cached,
|
||||
"executionInfo": {"cachedRemotely": cached_remotely, "strategy": "remote"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ReadShardDurationsTest(unittest.TestCase):
|
||||
def test_collects_executed_durations_per_shard(self):
|
||||
path = write_bep([
|
||||
test_result_event("//x:t", 1, 10_000),
|
||||
test_result_event("//x:t", 2, 20_000),
|
||||
])
|
||||
try:
|
||||
got = shard_budget.read_shard_durations(path)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
self.assertEqual(got, {"//x:t": {1: 10_000, 2: 20_000}})
|
||||
|
||||
def test_excludes_cached_results(self):
|
||||
path = write_bep([
|
||||
test_result_event("//x:t", 1, 10_000),
|
||||
test_result_event("//x:t", 2, 999_000, cached_remotely=True),
|
||||
])
|
||||
try:
|
||||
got = shard_budget.read_shard_durations(path)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
# Shard 2 was a cache hit: it contributes no duration.
|
||||
self.assertEqual(got, {"//x:t": {1: 10_000}})
|
||||
|
||||
def test_unsharded_defaults_to_shard_1(self):
|
||||
path = write_bep([test_result_event("//x:t", None, 5_000)])
|
||||
try:
|
||||
got = shard_budget.read_shard_durations(path)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
self.assertEqual(got, {"//x:t": {1: 5_000}})
|
||||
|
||||
def test_ignores_non_test_result_lines(self):
|
||||
path = write_bep([{"id": {"progress": {}}, "progress": {}}])
|
||||
try:
|
||||
got = shard_budget.read_shard_durations(path)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
self.assertEqual(got, {})
|
||||
|
||||
def test_malformed_line_is_skipped(self):
|
||||
f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8")
|
||||
f.write("not json\n")
|
||||
f.write(json.dumps(test_result_event("//x:t", 1, 1_000)) + "\n")
|
||||
f.close()
|
||||
try:
|
||||
got = shard_budget.read_shard_durations(f.name)
|
||||
finally:
|
||||
os.unlink(f.name)
|
||||
self.assertEqual(got, {"//x:t": {1: 1_000}})
|
||||
|
||||
|
||||
class EvaluateTest(unittest.TestCase):
|
||||
def test_balanced_shards_do_not_warn(self):
|
||||
# gascity #7241-shaped data: every shard within a few seconds of the
|
||||
# others never crosses max(150s, 2x median).
|
||||
results = shard_budget.evaluate({"//x:t": {i: 60_000 + i * 1000 for i in range(1, 11)}})
|
||||
self.assertEqual(len(results), 1)
|
||||
r = results[0]
|
||||
self.assertFalse(r.warn)
|
||||
self.assertFalse(r.fail)
|
||||
|
||||
def test_single_long_pole_warns(self):
|
||||
# gascity acceptance shard 4 shape: one shard far above the rest.
|
||||
shards = {i: 80_000 for i in range(1, 10)}
|
||||
shards[4] = 725_000
|
||||
results = shard_budget.evaluate({"//acceptance:t": shards})
|
||||
r = results[0]
|
||||
self.assertTrue(r.warn)
|
||||
self.assertEqual(r.slowest_shard, 4)
|
||||
self.assertAlmostEqual(r.slowest_ms, 725_000)
|
||||
|
||||
def test_fail_threshold_is_stricter_than_warn(self):
|
||||
# beads server-Dolt storage tier shape: a straggler 4.3-4.5 min
|
||||
# behind an otherwise ~80s median. Over the warn floor (150s, 2x
|
||||
# median) but under the fail floor (300s, 3x median) until it grows
|
||||
# further.
|
||||
shards = {i: 80_000 for i in range(1, 16)}
|
||||
shards[16] = 250_000
|
||||
r = shard_budget.evaluate({"//storage/dolt:t": shards})[0]
|
||||
self.assertTrue(r.warn)
|
||||
self.assertFalse(r.fail)
|
||||
|
||||
shards[16] = 400_000
|
||||
r = shard_budget.evaluate({"//storage/dolt:t": shards})[0]
|
||||
self.assertTrue(r.warn)
|
||||
self.assertTrue(r.fail)
|
||||
|
||||
def test_floor_applies_below_tiny_medians(self):
|
||||
# A fast, lopsided split (two 5s shards, one 200s shard) must still
|
||||
# warn even though 2x the 5s median is nowhere near the slowest
|
||||
# shard; the 150s floor catches it, not the ratio.
|
||||
r = shard_budget.evaluate({"//x:t": {1: 5_000, 2: 5_000, 3: 200_000}})[0]
|
||||
self.assertTrue(r.warn)
|
||||
|
||||
def test_fewer_than_two_shards_is_skipped(self):
|
||||
self.assertEqual(shard_budget.evaluate({"//x:t": {1: 10_000}}), [])
|
||||
self.assertEqual(shard_budget.evaluate({"//x:t": {}}), [])
|
||||
|
||||
def test_sorted_slowest_first(self):
|
||||
results = shard_budget.evaluate({
|
||||
"//a:t": {1: 10_000, 2: 20_000},
|
||||
"//b:t": {1: 10_000, 2: 500_000},
|
||||
})
|
||||
self.assertEqual([r.label for r in results], ["//b:t", "//a:t"])
|
||||
|
||||
|
||||
class MainTest(unittest.TestCase):
|
||||
def test_warn_only_exits_zero(self):
|
||||
shards = {i: 80_000 for i in range(1, 10)}
|
||||
shards[4] = 900_000
|
||||
events = [test_result_event("//x:t", i, ms) for i, ms in shards.items()]
|
||||
path = write_bep(events)
|
||||
try:
|
||||
rc = shard_budget.main(["--bep", path])
|
||||
finally:
|
||||
os.unlink(path)
|
||||
self.assertEqual(rc, 0)
|
||||
|
||||
def test_fail_on_imbalance_exits_nonzero_past_fail_threshold(self):
|
||||
shards = {i: 80_000 for i in range(1, 10)}
|
||||
shards[4] = 900_000
|
||||
events = [test_result_event("//x:t", i, ms) for i, ms in shards.items()]
|
||||
path = write_bep(events)
|
||||
try:
|
||||
rc = shard_budget.main(["--bep", path, "--fail-on-imbalance"])
|
||||
finally:
|
||||
os.unlink(path)
|
||||
self.assertEqual(rc, 1)
|
||||
|
||||
def test_fail_on_imbalance_still_zero_under_fail_threshold(self):
|
||||
shards = {i: 80_000 for i in range(1, 10)}
|
||||
shards[4] = 200_000
|
||||
events = [test_result_event("//x:t", i, ms) for i, ms in shards.items()]
|
||||
path = write_bep(events)
|
||||
try:
|
||||
rc = shard_budget.main(["--bep", path, "--fail-on-imbalance"])
|
||||
finally:
|
||||
os.unlink(path)
|
||||
self.assertEqual(rc, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue