fix(bazel): re-vendor the BEP test-timing clamp from gascity (extractor 1.1.1) (#7477)

#7374 re-vendored gascity's signed-duration fix for exec-log spawns, but
build_tests() still kept the sign of BEP testResult timings. rbe-west
scheduler/worker clock skew makes a remotely executed test report
queueTime -1 ms (-24 ms at worst), and the collector rejected the whole
artifact on tests[].results[].queue_ms: about 19% of beads artifacts,
holding 52% of beads' CI worker time.

Byte copy of gascity 2ddde0d94be15d95086cc2bcb169303ea1243456: clamp
duration_ms and every timingBreakdown child with _clamp_duration_ms.
Updates the two sha256 pins; testdata/ci_analytics is unchanged.

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Julian Knutsen 2026-10-10 16:20:37 +00:00 • committed by GitHub
commit a945619b5e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 76 additions and 11 deletions

View file

@ -18,13 +18,18 @@ import (
// fix on top of the final squashed-to-one-commit S2 history at
// 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac: negative protobuf durations were
// decoded as unsigned, so the collector rejected roughly 22% of artifacts on
// remote_queue_ms. The fixtures are generated under env -i and committed
// remote_queue_ms. Re-vendored at gascity commit
// 2ddde0d94be15d95086cc2bcb169303ea1243456
// (fix/ci-analytics-clamp-test-timings, extractor 1.1.1): BEP test timings
// kept their sign, so a remote test's -1 ms queueTime (clock skew) still
// rejected 17-32% of artifacts on tests[].results[].queue_ms. The fixtures
// are generated under env -i and committed
// with the compact exec log decompressed (*.binpb), never a zstd frame, so
// nothing opaque reaches version control, the same way
// tools/rbe/cache-zstd-probe.sh is a byte copy. gascity is canonical for
// this pair; TestCIAnalyticsExtractorIsPinnedToGascity below pins both
// files' sha256 so a local edit here is caught instead of silently
// drifting out of lockstep. The vendored test file's own 67 cases (run in
// drifting out of lockstep. The vendored test file's own 70 cases (run in
// place by TestCIAnalyticsExtractUnitTests) now include the
// mismatched-invocation-id and truncated-BEP cases beads originally added
// (test_beads_mismatch_fixture_reports_mismatch,
@ -43,11 +48,11 @@ func ciAnalyticsTestdataDir(root string) string {
}
// ciAnalyticsExtractSHA256 and ciAnalyticsExtractUnitTestSHA256 pin the
// vendored files' content, computed from gascity's signed-durations fix at
// 9865cc045aee1391dbcc7eac99339c26e3a5ce71.
// vendored files' content, computed from gascity's test-timing clamp at
// 2ddde0d94be15d95086cc2bcb169303ea1243456 (extractor 1.1.1).
const (
ciAnalyticsExtractSHA256 = "be6d187804f8b44a75af40113fef97c37476923a1c6a0966d6216a791935c674"
ciAnalyticsExtractUnitTestSHA256 = "7e67b54b0e6d374efed899c8cae448f75effcdd1210b053e242a3ff51f1c8992"
ciAnalyticsExtractSHA256 = "1d0049bc37d957a73c6c6f808d6103ddad290a8dd053980ec9f97d4750bed2b7"
ciAnalyticsExtractUnitTestSHA256 = "42331bf3e5ce5e650f62f7c00913b0155416352360e3b1d99be58e5508d22f9f"
// ciAnalyticsTestdataSHA256 pins testdata/ci_analytics/** as a whole
// (regen.sh, mismatch.bep.jsonl, poisoned-bep.json and real/*): a sha256
// over every file's slash-joined relative path and bytes, each
@ -57,7 +62,8 @@ const (
// individually-edited fixture that a single-file sha256 (the two
// above) can't. The signed-durations fix (9865cc045aee) touches only
// the extractor and its unit test, not testdata/ci_analytics/, so this
// pin is unchanged and was re-verified against gascity's tree.
// pin is unchanged and was re-verified against gascity's tree. The
// test-timing clamp (2ddde0d94be1) likewise leaves testdata alone.
ciAnalyticsTestdataSHA256 = "dc9ac595df095ef8fe1b965d0e2a8d0b0e5694c3122208891d27655bcc2285d5"
)
@ -120,7 +126,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) {
}
sum := sha256.Sum256(raw)
if got := hex.EncodeToString(sum[:]); got != tc.want {
t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extract-signed-durations @ 9865cc045aee1391dbcc7eac99339c26e3a5ce71): "+
t.Errorf("%s sha256 = %s, want %s (gascity fix/ci-analytics-clamp-test-timings @ 2ddde0d94be15d95086cc2bcb169303ea1243456, extractor 1.1.1): "+
"either this file drifted from gascity's canonical copy, or gascity changed it "+
"and this pin (and the vendored copy) needs updating to match", tc.path, got, tc.want)
}
@ -133,7 +139,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) {
}
}
// TestCIAnalyticsExtractUnitTests runs gascity's own 67 unittest cases
// TestCIAnalyticsExtractUnitTests runs gascity's own 70 unittest cases
// in-place against the vendored extractor: the real Bazel 9.2.0 fixtures
// (testdata/ci_analytics/real/*, regenerated under env -i after the S2
// leak), the synthetic remote/cache-hit exec log built by the test's own

View file

@ -63,8 +63,9 @@ SCHEMA_VERSION = 1
# Bumped alongside the signed-duration fix below, so S4 can tell a
# fixed-up artifact (no more 2^63/2^64-scale decoy durations from a
# negative queue/exec/setup/upload time decoded as unsigned) from one
# produced by the earlier, buggy extractor.
EXTRACTOR_VERSION = "1.1.0"
# produced by the earlier, buggy extractor. 1.1.1 also clamps BEP test
# timings (build_tests), so a skewed remote test no longer reports -1 ms.
EXTRACTOR_VERSION = "1.1.1"
# A per-spawn duration (Duration.seconds*1000 + Duration.nanos//1e6, after
# signed decoding) above this is treated as corrupt, not a real build: a
@ -961,6 +962,10 @@ def build_tests(events):
duration_ms = as_duration_ms(tr.get("testAttemptDuration"))
if duration_ms is None:
duration_ms = as_int(tr.get("testAttemptDurationMillis"))
# BEP Durations keep their sign. rbe-west's scheduler and a worker disagree by about
# a millisecond, so a remotely executed test reports queueTime "-0.000612s" (-1 ms):
# clamp like dur_ms() does for exec-log spawns (the collector rejects negatives).
duration_ms = _clamp_duration_ms(duration_ms)
queue_ms = fetch_ms = exec_ms = 0
# Per-test timing lives under testResult.executionInfo.
# timingBreakdown, not a top-level testResult.timingBreakdown
@ -975,6 +980,7 @@ def build_tests(events):
ms = as_duration_ms(child.get("time"))
if ms is None:
continue
ms = _clamp_duration_ms(ms)
if name == "queueTime":
queue_ms = ms
elif name == "fetchTime":

View file

@ -1528,5 +1528,58 @@ class BEPReadContractTests(unittest.TestCase):
self.assertEqual(rec["mnemonics"], [])
class NegativeTestTimingTests(unittest.TestCase):
"""rbe-west scheduler/worker clock skew: a remotely executed test's BEP
timingBreakdown reports a slightly negative queueTime. Extractor 1.1.0
kept the sign, and the collector rejected the whole artifact on
$.invocations[].tests[].results[].queue_ms (1,131 artifacts on
2026-10-08..10, all from remote test results, -1 to -24 ms)."""
@staticmethod
def _result(children, duration="1.5s"):
ev = {
"id": {"testResult": {"label": "//pkg:t", "shard": 1, "run": 1, "attempt": 1}},
"testResult": {
"status": "PASSED",
"testAttemptDuration": duration,
"executionInfo": {
"strategy": "remote",
"timingBreakdown": {"child": children},
},
},
}
tests, _ = extract.build_tests([ev])
return tests[0]["results"][0]
def test_skewed_remote_test_timing_clamps_to_zero(self):
r = self._result(
[
{"name": "queueTime", "time": "-0.000612s"},
{"name": "fetchTime", "time": "-0.002s"},
{"name": "executionWallTime", "time": "1.2s"},
]
)
self.assertEqual(r["runner"], "remote")
self.assertEqual(
(r["queue_ms"], r["fetch_ms"], r["exec_ms"], r["duration_ms"]), (0, 0, 1200, 1500)
)
def test_worst_observed_skew_and_negative_duration_clamp_to_zero(self):
# -24 ms is the most negative queue_ms in the rejected artifacts.
r = self._result(
[
{"name": "queueTime", "time": "-0.024s"},
{"name": "executionWallTime", "time": "-1s"},
],
duration="-0.5s",
)
self.assertEqual((r["queue_ms"], r["exec_ms"], r["duration_ms"]), (0, 0, 0))
def test_implausibly_large_test_timing_clamps_to_zero(self):
too_big = "%ds" % (extract.MAX_SANE_DURATION_MS // 1000 + 1)
r = self._result([{"name": "queueTime", "time": too_big}], duration=too_big)
self.assertEqual((r["queue_ms"], r["duration_ms"]), (0, 0))
if __name__ == "__main__":
unittest.main()