From df72811377fb212b90647f91563c68a367bb9911 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 00:56:53 +0000 Subject: [PATCH] fix(bazel): re-vendor signed-duration decode fix from gascity gascity commit 9865cc045aee1391dbcc7eac99339c26e3a5ce71 fixed a production bug: negative protobuf durations in the compact exec log were decoded as unsigned, so the collector rejected roughly 22% of artifacts on remote_queue_ms. Re-vendor the extractor and its unit test byte-identical from that commit and update the sha256 pins in ci_analytics_extract_test.go accordingly. testdata/ci_analytics/ is untouched by the fix; its tree-hash pin is re-verified unchanged against gascity's tree. Co-Authored-By: Claude Sonnet 5 --- scripts/ci_analytics_extract_test.go | 34 ++++---- tools/bazel/ci_analytics_extract.py | 77 +++++++++++++++-- tools/bazel/ci_analytics_extract_test.py | 103 +++++++++++++++++++++++ 3 files changed, 190 insertions(+), 24 deletions(-) diff --git a/scripts/ci_analytics_extract_test.go b/scripts/ci_analytics_extract_test.go index db5fc463b..694276db8 100644 --- a/scripts/ci_analytics_extract_test.go +++ b/scripts/ci_analytics_extract_test.go @@ -13,18 +13,18 @@ import ( // ciAnalyticsExtractScript and ciAnalyticsExtractUnitTest are a byte copy of // gascity's tools/bazel/ci_analytics_extract.py and -// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extractor, -// gascity commit 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac, the final -// squashed-to-one-commit history superseding 32d51ce26a and the original -// 9104311a8a after the S2 review found the first cut's real/*-bep.json -// fixtures leaked operator env: a token, SSH IPs, the OpenBao address, the -// hostname and home paths. The fixtures are generated under env -i and -// committed with the compact exec log decompressed (*.binpb), never a -// zstd frame, so nothing opaque reaches version control), the same way +// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extract-signed-durations, +// gascity commit 9865cc045aee1391dbcc7eac99339c26e3a5ce71, a production bug +// fix on top of the final squashed-to-one-commit S2 history at +// 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac: negative protobuf durations were +// decoded as unsigned, so the collector rejected roughly 22% of artifacts on +// remote_queue_ms. The fixtures are generated under env -i and committed +// with the compact exec log decompressed (*.binpb), never a zstd frame, so +// nothing opaque reaches version control, the same way // tools/rbe/cache-zstd-probe.sh is a byte copy. gascity is canonical for // this pair; TestCIAnalyticsExtractorIsPinnedToGascity below pins both // files' sha256 so a local edit here is caught instead of silently -// drifting out of lockstep. The vendored test file's own 62 cases (run in +// drifting out of lockstep. The vendored test file's own 67 cases (run in // place by TestCIAnalyticsExtractUnitTests) now include the // mismatched-invocation-id and truncated-BEP cases beads originally added // (test_beads_mismatch_fixture_reports_mismatch, @@ -43,11 +43,11 @@ func ciAnalyticsTestdataDir(root string) string { } // ciAnalyticsExtractSHA256 and ciAnalyticsExtractUnitTestSHA256 pin the -// vendored files' content, computed from gascity's final sanitized, -// squashed-to-one-commit history at 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac. +// vendored files' content, computed from gascity's signed-durations fix at +// 9865cc045aee1391dbcc7eac99339c26e3a5ce71. const ( - ciAnalyticsExtractSHA256 = "6bb912b17faac59a12ec53f71fbfbedfecc503037b6f5954e878d108abaae44f" - ciAnalyticsExtractUnitTestSHA256 = "e833e1c961010cd02189afaef273a261a54ccabe916b03bae3115177378a405b" + ciAnalyticsExtractSHA256 = "be6d187804f8b44a75af40113fef97c37476923a1c6a0966d6216a791935c674" + ciAnalyticsExtractUnitTestSHA256 = "7e67b54b0e6d374efed899c8cae448f75effcdd1210b053e242a3ff51f1c8992" // ciAnalyticsTestdataSHA256 pins testdata/ci_analytics/** as a whole // (regen.sh, mismatch.bep.jsonl, poisoned-bep.json and real/*): a sha256 // over every file's slash-joined relative path and bytes, each @@ -55,7 +55,9 @@ const ( // computed the same way over gascity's 00161373bc tree and confirmed // identical, so this one hash catches a dropped, renamed or // individually-edited fixture that a single-file sha256 (the two - // above) can't. + // above) can't. The signed-durations fix (9865cc045aee) touches only + // the extractor and its unit test, not testdata/ci_analytics/, so this + // pin is unchanged and was re-verified against gascity's tree. ciAnalyticsTestdataSHA256 = "dc9ac595df095ef8fe1b965d0e2a8d0b0e5694c3122208891d27655bcc2285d5" ) @@ -118,7 +120,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) { } sum := sha256.Sum256(raw) if got := hex.EncodeToString(sum[:]); got != tc.want { - t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac): "+ + t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extract-signed-durations @ 9865cc045aee1391dbcc7eac99339c26e3a5ce71): "+ "either this file drifted from gascity's canonical copy, or gascity changed it "+ "and this pin (and the vendored copy) needs updating to match", tc.path, got, tc.want) } @@ -131,7 +133,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) { } } -// TestCIAnalyticsExtractUnitTests runs gascity's own 62 unittest cases +// TestCIAnalyticsExtractUnitTests runs gascity's own 67 unittest cases // in-place against the vendored extractor: the real Bazel 9.2.0 fixtures // (testdata/ci_analytics/real/*, regenerated under env -i after the S2 // leak), the synthetic remote/cache-hit exec log built by the test's own diff --git a/tools/bazel/ci_analytics_extract.py b/tools/bazel/ci_analytics_extract.py index 7f5d9bc3d..7696b2e75 100644 --- a/tools/bazel/ci_analytics_extract.py +++ b/tools/bazel/ci_analytics_extract.py @@ -60,7 +60,18 @@ import time FORMAT = "ci-analytics-summary" SCHEMA_VERSION = 1 -EXTRACTOR_VERSION = "1.0.0" +# Bumped alongside the signed-duration fix below, so S4 can tell a +# fixed-up artifact (no more 2^63/2^64-scale decoy durations from a +# negative queue/exec/setup/upload time decoded as unsigned) from one +# produced by the earlier, buggy extractor. +EXTRACTOR_VERSION = "1.1.0" + +# A per-spawn duration (Duration.seconds*1000 + Duration.nanos//1e6, after +# signed decoding) above this is treated as corrupt, not a real build: a +# single spawn cannot legitimately run for more than a day. Also the upper +# clamp for the other duration/int fields this file decodes (see +# _clamp_duration_ms below). +MAX_SANE_DURATION_MS = 24 * 60 * 60 * 1000 MAX_INVOCATIONS = 8 MAX_MNEMONICS = 200 @@ -368,17 +379,56 @@ def fields(b, deadline=None): yield f, t, v +def _to_signed64(v): + """varint()/fields() decode a wire-type-0 field's raw bits as an + unsigned 64-bit value; protobuf's int64 (and int32 -- see + _to_signed32) are the two's-complement bit pattern of a *signed* + value, re-encoded with the same unsigned-varint algorithm. A real + negative int64 (e.g. a Duration whose seconds/nanos went negative from + clock skew between the NativeLink scheduler and a worker) is written + as a 10-byte varint of its 64-bit two's complement form, which this + recovers: values with the top bit set (>= 2^63) are negative.""" + return v - (1 << 64) if v >= (1 << 63) else v + + +def _to_signed32(v): + """Same, for an int32 field (Duration.nanos): proto encoders sign- + extend a negative int32 to 64 bits before varint-encoding it, so + _to_signed64(v) alone already recovers the right numeric value for a + well-formed input. This additionally truncates to the low 32 bits + first, so a corrupt/hostile encoder that puts a value outside the + int32 range on an int32-typed field degrades to *some* signed 32-bit + value instead of silently keeping 64 bits of it.""" + v &= 0xFFFFFFFF + return v - (1 << 32) if v >= (1 << 31) else v + + +def _clamp_duration_ms(ms): + """Clamps a decoded duration to [0, MAX_SANE_DURATION_MS]. A negative + duration -- the whole point of this fix: clock skew between the + NativeLink scheduler and a worker can make a remote spawn's queue time + read slightly negative, which _to_signed64() now decodes correctly + instead of as a near-2^63 positive decoy -- counts as 0, and so does + anything implausibly large (not a real build, so probably corrupt). + Applied per spawn/per field, before any sum: one poisoned spawn can + never poison a mnemonic's or label's total.""" + if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0 or ms > MAX_SANE_DURATION_MS: + return 0 + return ms + + def dur_ms(v, deadline=None): - """Decodes a Duration submessage's bytes (fields 1 seconds, 2 nanos).""" + """Decodes a Duration submessage's bytes (fields 1 seconds, 2 nanos) to + a signed-then-clamped millisecond count.""" seconds = nanos = 0 for f, t, val in fields(v, deadline): if t != 0: continue if f == 1: - seconds = val + seconds = _to_signed64(val) elif f == 2: - nanos = val - return seconds * 1000 + nanos // 1_000_000 + nanos = _to_signed32(val) + return _clamp_duration_ms(seconds * 1000 + nanos // 1_000_000) def spawns(raw, deadline): @@ -425,7 +475,12 @@ def spawns(raw, deadline): if mf in SPAWN_METRIC and mt == 2 and isinstance(mv, bytes): s["m"][SPAWN_METRIC[mf]] = dur_ms(mv, deadline) elif mf == 11 and mt == 0: - s["m"]["input_bytes"] = mv + # Also int64; not a duration, but still + # decoded as unsigned bits -- a negative + # value here is nonsensical, so clamp to 0 + # rather than letting it decode to a huge + # positive "byte count" either. + s["m"]["input_bytes"] = max(0, _to_signed64(mv)) yield inv, s @@ -858,8 +913,14 @@ def build_action_data(events): ) d["bep_executed"] += as_int(ad.get("actionsExecuted")) d["bep_created"] += as_int(ad.get("actionsCreated")) - d["user_ms"] += as_duration_ms(ad.get("userTime")) or 0 - d["system_ms"] += as_duration_ms(ad.get("systemTime")) or 0 + # BEP's own Duration strings ("3.14s") aren't subject to the + # unsigned-varint bug (DURATION_RE allows a leading "-", so + # a legitimately negative one would already parse), but the + # same sane-duration clamp is applied here too, for the same + # reason: one corrupt/adversarial value should never poison + # this mnemonic's running total. + d["user_ms"] += _clamp_duration_ms(as_duration_ms(ad.get("userTime")) or 0) + d["system_ms"] += _clamp_duration_ms(as_duration_ms(ad.get("systemTime")) or 0) except Exception: continue return data diff --git a/tools/bazel/ci_analytics_extract_test.py b/tools/bazel/ci_analytics_extract_test.py index dc7de9205..6f55f0bba 100644 --- a/tools/bazel/ci_analytics_extract_test.py +++ b/tools/bazel/ci_analytics_extract_test.py @@ -79,6 +79,27 @@ def _duration_bytes(ms: int) -> bytes: return _varint_field(1, seconds) + _varint_field(2, nanos * 1_000_000) +def _signed_varint(n: int) -> bytes: + """Encodes a signed int64/int32 field's value the way a real protobuf + encoder does: the 64-bit two's-complement bit pattern, varint-encoded + as if it were unsigned -- a negative value always takes the full 10 + bytes. _varint() itself can't be used directly for a negative n: `n & + 0x7F`/`n >>= 7` on a negative Python int never reaches 0 (arithmetic + shift preserves the sign), so it would loop forever.""" + return _varint(n & 0xFFFFFFFFFFFFFFFF) + + +def _signed_varint_field(field: int, n: int) -> bytes: + return _tag(field, 0) + _signed_varint(n) + + +def _signed_duration_bytes(seconds: int, nanos: int) -> bytes: + """Builds a Duration submessage (fields 1 seconds, 2 nanos) with + explicit, possibly-negative values, each encoded the way a real + negative int64/int32 would be: a 10-byte two's-complement varint.""" + return _signed_varint_field(1, seconds) + _signed_varint_field(2, nanos) + + def _spawn_metrics(total_ms=0, queue_ms=0, exec_ms=0, input_bytes=0) -> bytes: body = b"" if total_ms: @@ -293,6 +314,31 @@ class ExtractorHelperTests(unittest.TestCase): self.assertEqual(extract._clamp_pct(float("-inf")), 0.0) self.assertEqual(extract._clamp_pct(float("nan")), 0.0) + def test_to_signed64_recovers_negative_values_from_the_raw_varint_bits(self): + # -1 as a 64-bit two's complement varint is 10 bytes of 0xFF-ish + # bits; varint() decodes that as 2**64 - 1. + self.assertEqual(extract._to_signed64((1 << 64) - 1), -1) + self.assertEqual(extract._to_signed64(1 << 63), -(1 << 63)) + self.assertEqual(extract._to_signed64(0), 0) + self.assertEqual(extract._to_signed64((1 << 63) - 1), (1 << 63) - 1) + + def test_to_signed32_truncates_then_sign_extends(self): + self.assertEqual(extract._to_signed32((1 << 32) - 1), -1) + self.assertEqual(extract._to_signed32(1 << 31), -(1 << 31)) + self.assertEqual(extract._to_signed32(0), 0) + # A hostile/corrupt int64-range value on an int32-typed field is + # truncated to the low 32 bits first, not kept as a huge int64. + self.assertEqual(extract._to_signed32((1 << 40) + 5), 5) + + def test_clamp_duration_ms_rejects_negative_and_absurdly_large(self): + self.assertEqual(extract._clamp_duration_ms(500), 500) + self.assertEqual(extract._clamp_duration_ms(0), 0) + self.assertEqual(extract._clamp_duration_ms(-1500), 0) + self.assertEqual(extract._clamp_duration_ms(extract.MAX_SANE_DURATION_MS), extract.MAX_SANE_DURATION_MS) + self.assertEqual(extract._clamp_duration_ms(extract.MAX_SANE_DURATION_MS + 1), 0) + self.assertEqual(extract._clamp_duration_ms(float("inf")), 0) + self.assertEqual(extract._clamp_duration_ms(float("nan")), 0) + class RealFixtureTests(unittest.TestCase): """testdata/ci_analytics/real/*: genuine Bazel 9.2.0 output.""" @@ -1161,6 +1207,63 @@ class RobustnessTests(unittest.TestCase): self.assertEqual(status, "corrupt") self.assertEqual([r["repo"] for r in rows], ["@@rules_cc+"]) + def test_negative_queue_duration_decodes_to_zero_not_a_huge_positive(self): + """The production bug: clock skew between the NativeLink scheduler + and a worker can make a remote spawn's queue time read slightly + negative. protobuf encodes that as a 10-byte two's-complement + varint of the Duration's seconds/nanos; decoding those bits as + unsigned (the pre-fix behavior) produced "remote_queue_ms" values + around 2**63/1e3 -- not a small negative number. Built with the + test-side signed varint encoder (_signed_duration_bytes), not the + plain _duration_bytes() helper, which can't represent a negative + value at all.""" + uuid = "11111111-2222-3333-4444-555555555555" + # -1.5s of (impossible, but exactly what clock skew would produce) + # negative queue time, alongside a normal positive total/exec so + # the fix's "other values intact" half is also checked. + metrics = ( + _bytes_field(5, _signed_duration_bytes(-1, -500_000_000)) + + _bytes_field(1, _duration_bytes(200)) + + _bytes_field(8, _duration_bytes(50)) + ) + spawn_body = _spawn("//:synth_remote2", "TestRunner", "remote", False, metrics) + exec_log = _invocation_entry(uuid) + _entry_with_spawn(spawn_body) + rec = self._run(self._bep(uuid), exec_log) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "ok") + by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]} + tr = by_mnemonic["TestRunner"] + self.assertEqual(tr["remote_queue_ms"], 0) + self.assertEqual(tr["remote_queue_max_ms"], 0) + # Not poisoned by the clamped-away queue value: total/exec, decoded + # from the same spawn's other, non-negative Duration fields, are + # unaffected. + self.assertEqual(tr["total_ms"], 200) + self.assertEqual(tr["remote_exec_ms"], 50) + + def test_absurdly_large_duration_clamps_to_zero(self): + """A corrupt (not just adversarial) producer could put an + implausible but individually well-formed value on a Duration + field -- MAX_SANE_DURATION_MS guards against that the same way as + the negative case, so one bad spawn can't blow out a mnemonic's + sum in either direction.""" + uuid = "11111111-2222-3333-4444-555555555555" + absurd_ms = 50 * 60 * 60 * 1000 # 50 hours: no real spawn runs this long + metrics = ( + _bytes_field(5, _duration_bytes(absurd_ms)) + + _bytes_field(1, _duration_bytes(200)) + + _bytes_field(8, _duration_bytes(50)) + ) + spawn_body = _spawn("//:synth_remote3", "TestRunner", "remote", False, metrics) + exec_log = _invocation_entry(uuid) + _entry_with_spawn(spawn_body) + rec = self._run(self._bep(uuid), exec_log) + self.assertIsNotNone(rec) + by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]} + tr = by_mnemonic["TestRunner"] + self.assertEqual(tr["remote_queue_ms"], 0) + self.assertEqual(tr["total_ms"], 200) + self.assertEqual(tr["remote_exec_ms"], 50) + class BoundsTests(unittest.TestCase): def test_exec_log_parse_timeout_is_reported(self):