fix(bazel): re-vendor the BEP test-timing clamp from gascity (extractor 1.1.1) (#7477)
#7374 re-vendored gascity's signed-duration fix for exec-log spawns, but build_tests() still kept the sign of BEP testResult timings. rbe-west scheduler/worker clock skew makes a remotely executed test report queueTime -1 ms (-24 ms at worst), and the collector rejected the whole artifact on tests[].results[].queue_ms: about 19% of beads artifacts, holding 52% of beads' CI worker time. Byte copy of gascity 2ddde0d94be15d95086cc2bcb169303ea1243456: clamp duration_ms and every timingBreakdown child with _clamp_duration_ms. Updates the two sha256 pins; testdata/ci_analytics is unchanged. Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
6c1848a77e
commit
a945619b5e
3 changed files with 76 additions and 11 deletions
|
|
@ -18,13 +18,18 @@ import (
|
|||
// fix on top of the final squashed-to-one-commit S2 history at
|
||||
// 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac: negative protobuf durations were
|
||||
// decoded as unsigned, so the collector rejected roughly 22% of artifacts on
|
||||
// remote_queue_ms. The fixtures are generated under env -i and committed
|
||||
// remote_queue_ms. Re-vendored at gascity commit
|
||||
// 2ddde0d94be15d95086cc2bcb169303ea1243456
|
||||
// (fix/ci-analytics-clamp-test-timings, extractor 1.1.1): BEP test timings
|
||||
// kept their sign, so a remote test's -1 ms queueTime (clock skew) still
|
||||
// rejected 17-32% of artifacts on tests[].results[].queue_ms. The fixtures
|
||||
// are generated under env -i and committed
|
||||
// with the compact exec log decompressed (*.binpb), never a zstd frame, so
|
||||
// nothing opaque reaches version control, the same way
|
||||
// tools/rbe/cache-zstd-probe.sh is a byte copy. gascity is canonical for
|
||||
// this pair; TestCIAnalyticsExtractorIsPinnedToGascity below pins both
|
||||
// files' sha256 so a local edit here is caught instead of silently
|
||||
// drifting out of lockstep. The vendored test file's own 67 cases (run in
|
||||
// drifting out of lockstep. The vendored test file's own 70 cases (run in
|
||||
// place by TestCIAnalyticsExtractUnitTests) now include the
|
||||
// mismatched-invocation-id and truncated-BEP cases beads originally added
|
||||
// (test_beads_mismatch_fixture_reports_mismatch,
|
||||
|
|
@ -43,11 +48,11 @@ func ciAnalyticsTestdataDir(root string) string {
|
|||
}
|
||||
|
||||
// ciAnalyticsExtractSHA256 and ciAnalyticsExtractUnitTestSHA256 pin the
|
||||
// vendored files' content, computed from gascity's signed-durations fix at
|
||||
// 9865cc045aee1391dbcc7eac99339c26e3a5ce71.
|
||||
// vendored files' content, computed from gascity's test-timing clamp at
|
||||
// 2ddde0d94be15d95086cc2bcb169303ea1243456 (extractor 1.1.1).
|
||||
const (
|
||||
ciAnalyticsExtractSHA256 = "be6d187804f8b44a75af40113fef97c37476923a1c6a0966d6216a791935c674"
|
||||
ciAnalyticsExtractUnitTestSHA256 = "7e67b54b0e6d374efed899c8cae448f75effcdd1210b053e242a3ff51f1c8992"
|
||||
ciAnalyticsExtractSHA256 = "1d0049bc37d957a73c6c6f808d6103ddad290a8dd053980ec9f97d4750bed2b7"
|
||||
ciAnalyticsExtractUnitTestSHA256 = "42331bf3e5ce5e650f62f7c00913b0155416352360e3b1d99be58e5508d22f9f"
|
||||
// ciAnalyticsTestdataSHA256 pins testdata/ci_analytics/** as a whole
|
||||
// (regen.sh, mismatch.bep.jsonl, poisoned-bep.json and real/*): a sha256
|
||||
// over every file's slash-joined relative path and bytes, each
|
||||
|
|
@ -57,7 +62,8 @@ const (
|
|||
// individually-edited fixture that a single-file sha256 (the two
|
||||
// above) can't. The signed-durations fix (9865cc045aee) touches only
|
||||
// the extractor and its unit test, not testdata/ci_analytics/, so this
|
||||
// pin is unchanged and was re-verified against gascity's tree.
|
||||
// pin is unchanged and was re-verified against gascity's tree. The
|
||||
// test-timing clamp (2ddde0d94be1) likewise leaves testdata alone.
|
||||
ciAnalyticsTestdataSHA256 = "dc9ac595df095ef8fe1b965d0e2a8d0b0e5694c3122208891d27655bcc2285d5"
|
||||
)
|
||||
|
||||
|
|
@ -120,7 +126,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) {
|
|||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
if got := hex.EncodeToString(sum[:]); got != tc.want {
|
||||
t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extract-signed-durations @ 9865cc045aee1391dbcc7eac99339c26e3a5ce71): "+
|
||||
t.Errorf("%s sha256 = %s, want %s (gascity fix/ci-analytics-clamp-test-timings @ 2ddde0d94be15d95086cc2bcb169303ea1243456, extractor 1.1.1): "+
|
||||
"either this file drifted from gascity's canonical copy, or gascity changed it "+
|
||||
"and this pin (and the vendored copy) needs updating to match", tc.path, got, tc.want)
|
||||
}
|
||||
|
|
@ -133,7 +139,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestCIAnalyticsExtractUnitTests runs gascity's own 67 unittest cases
|
||||
// TestCIAnalyticsExtractUnitTests runs gascity's own 70 unittest cases
|
||||
// in-place against the vendored extractor: the real Bazel 9.2.0 fixtures
|
||||
// (testdata/ci_analytics/real/*, regenerated under env -i after the S2
|
||||
// leak), the synthetic remote/cache-hit exec log built by the test's own
|
||||
|
|
|
|||
|
|
@ -63,8 +63,9 @@ SCHEMA_VERSION = 1
|
|||
# Bumped alongside the signed-duration fix below, so S4 can tell a
|
||||
# fixed-up artifact (no more 2^63/2^64-scale decoy durations from a
|
||||
# negative queue/exec/setup/upload time decoded as unsigned) from one
|
||||
# produced by the earlier, buggy extractor.
|
||||
EXTRACTOR_VERSION = "1.1.0"
|
||||
# produced by the earlier, buggy extractor. 1.1.1 also clamps BEP test
|
||||
# timings (build_tests), so a skewed remote test no longer reports -1 ms.
|
||||
EXTRACTOR_VERSION = "1.1.1"
|
||||
|
||||
# A per-spawn duration (Duration.seconds*1000 + Duration.nanos//1e6, after
|
||||
# signed decoding) above this is treated as corrupt, not a real build: a
|
||||
|
|
@ -961,6 +962,10 @@ def build_tests(events):
|
|||
duration_ms = as_duration_ms(tr.get("testAttemptDuration"))
|
||||
if duration_ms is None:
|
||||
duration_ms = as_int(tr.get("testAttemptDurationMillis"))
|
||||
# BEP Durations keep their sign. rbe-west's scheduler and a worker disagree by about
|
||||
# a millisecond, so a remotely executed test reports queueTime "-0.000612s" (-1 ms):
|
||||
# clamp like dur_ms() does for exec-log spawns (the collector rejects negatives).
|
||||
duration_ms = _clamp_duration_ms(duration_ms)
|
||||
queue_ms = fetch_ms = exec_ms = 0
|
||||
# Per-test timing lives under testResult.executionInfo.
|
||||
# timingBreakdown, not a top-level testResult.timingBreakdown
|
||||
|
|
@ -975,6 +980,7 @@ def build_tests(events):
|
|||
ms = as_duration_ms(child.get("time"))
|
||||
if ms is None:
|
||||
continue
|
||||
ms = _clamp_duration_ms(ms)
|
||||
if name == "queueTime":
|
||||
queue_ms = ms
|
||||
elif name == "fetchTime":
|
||||
|
|
|
|||
|
|
@ -1528,5 +1528,58 @@ class BEPReadContractTests(unittest.TestCase):
|
|||
self.assertEqual(rec["mnemonics"], [])
|
||||
|
||||
|
||||
class NegativeTestTimingTests(unittest.TestCase):
|
||||
"""rbe-west scheduler/worker clock skew: a remotely executed test's BEP
|
||||
timingBreakdown reports a slightly negative queueTime. Extractor 1.1.0
|
||||
kept the sign, and the collector rejected the whole artifact on
|
||||
$.invocations[].tests[].results[].queue_ms (1,131 artifacts on
|
||||
2026-10-08..10, all from remote test results, -1 to -24 ms)."""
|
||||
|
||||
@staticmethod
|
||||
def _result(children, duration="1.5s"):
|
||||
ev = {
|
||||
"id": {"testResult": {"label": "//pkg:t", "shard": 1, "run": 1, "attempt": 1}},
|
||||
"testResult": {
|
||||
"status": "PASSED",
|
||||
"testAttemptDuration": duration,
|
||||
"executionInfo": {
|
||||
"strategy": "remote",
|
||||
"timingBreakdown": {"child": children},
|
||||
},
|
||||
},
|
||||
}
|
||||
tests, _ = extract.build_tests([ev])
|
||||
return tests[0]["results"][0]
|
||||
|
||||
def test_skewed_remote_test_timing_clamps_to_zero(self):
|
||||
r = self._result(
|
||||
[
|
||||
{"name": "queueTime", "time": "-0.000612s"},
|
||||
{"name": "fetchTime", "time": "-0.002s"},
|
||||
{"name": "executionWallTime", "time": "1.2s"},
|
||||
]
|
||||
)
|
||||
self.assertEqual(r["runner"], "remote")
|
||||
self.assertEqual(
|
||||
(r["queue_ms"], r["fetch_ms"], r["exec_ms"], r["duration_ms"]), (0, 0, 1200, 1500)
|
||||
)
|
||||
|
||||
def test_worst_observed_skew_and_negative_duration_clamp_to_zero(self):
|
||||
# -24 ms is the most negative queue_ms in the rejected artifacts.
|
||||
r = self._result(
|
||||
[
|
||||
{"name": "queueTime", "time": "-0.024s"},
|
||||
{"name": "executionWallTime", "time": "-1s"},
|
||||
],
|
||||
duration="-0.5s",
|
||||
)
|
||||
self.assertEqual((r["queue_ms"], r["exec_ms"], r["duration_ms"]), (0, 0, 0))
|
||||
|
||||
def test_implausibly_large_test_timing_clamps_to_zero(self):
|
||||
too_big = "%ds" % (extract.MAX_SANE_DURATION_MS // 1000 + 1)
|
||||
r = self._result([{"name": "queueTime", "time": too_big}], duration=too_big)
|
||||
self.assertEqual((r["queue_ms"], r["duration_ms"]), (0, 0))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue