Compare commits

...
Sign in to create a new pull request.

1 commit

Author SHA1 Message Date
Julian Knutsen
df72811377 fix(bazel): re-vendor signed-duration decode fix from gascity
gascity commit 9865cc045aee1391dbcc7eac99339c26e3a5ce71 fixed a
production bug: negative protobuf durations in the compact exec log
were decoded as unsigned, so the collector rejected roughly 22% of
artifacts on remote_queue_ms. Re-vendor the extractor and its unit
test byte-identical from that commit and update the sha256 pins in
ci_analytics_extract_test.go accordingly. testdata/ci_analytics/ is
untouched by the fix; its tree-hash pin is re-verified unchanged
against gascity's tree.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-08 00:56:53 +00:00
3 changed files with 190 additions and 24 deletions

View file

@ -13,18 +13,18 @@ import (
// ciAnalyticsExtractScript and ciAnalyticsExtractUnitTest are a byte copy of
// gascity's tools/bazel/ci_analytics_extract.py and
// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extractor,
// gascity commit 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac, the final
// squashed-to-one-commit history superseding 32d51ce26a and the original
// 9104311a8a after the S2 review found the first cut's real/*-bep.json
// fixtures leaked operator env: a token, SSH IPs, the OpenBao address, the
// hostname and home paths. The fixtures are generated under env -i and
// committed with the compact exec log decompressed (*.binpb), never a
// zstd frame, so nothing opaque reaches version control), the same way
// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extract-signed-durations,
// gascity commit 9865cc045aee1391dbcc7eac99339c26e3a5ce71, a production bug
// fix on top of the final squashed-to-one-commit S2 history at
// 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac: negative protobuf durations were
// decoded as unsigned, so the collector rejected roughly 22% of artifacts on
// remote_queue_ms. The fixtures are generated under env -i and committed
// with the compact exec log decompressed (*.binpb), never a zstd frame, so
// nothing opaque reaches version control, the same way
// tools/rbe/cache-zstd-probe.sh is a byte copy. gascity is canonical for
// this pair; TestCIAnalyticsExtractorIsPinnedToGascity below pins both
// files' sha256 so a local edit here is caught instead of silently
// drifting out of lockstep. The vendored test file's own 62 cases (run in
// drifting out of lockstep. The vendored test file's own 67 cases (run in
// place by TestCIAnalyticsExtractUnitTests) now include the
// mismatched-invocation-id and truncated-BEP cases beads originally added
// (test_beads_mismatch_fixture_reports_mismatch,
@ -43,11 +43,11 @@ func ciAnalyticsTestdataDir(root string) string {
}
// ciAnalyticsExtractSHA256 and ciAnalyticsExtractUnitTestSHA256 pin the
// vendored files' content, computed from gascity's final sanitized,
// squashed-to-one-commit history at 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac.
// vendored files' content, computed from gascity's signed-durations fix at
// 9865cc045aee1391dbcc7eac99339c26e3a5ce71.
const (
ciAnalyticsExtractSHA256 = "6bb912b17faac59a12ec53f71fbfbedfecc503037b6f5954e878d108abaae44f"
ciAnalyticsExtractUnitTestSHA256 = "e833e1c961010cd02189afaef273a261a54ccabe916b03bae3115177378a405b"
ciAnalyticsExtractSHA256 = "be6d187804f8b44a75af40113fef97c37476923a1c6a0966d6216a791935c674"
ciAnalyticsExtractUnitTestSHA256 = "7e67b54b0e6d374efed899c8cae448f75effcdd1210b053e242a3ff51f1c8992"
// ciAnalyticsTestdataSHA256 pins testdata/ci_analytics/** as a whole
// (regen.sh, mismatch.bep.jsonl, poisoned-bep.json and real/*): a sha256
// over every file's slash-joined relative path and bytes, each
@ -55,7 +55,9 @@ const (
// computed the same way over gascity's 00161373bc tree and confirmed
// identical, so this one hash catches a dropped, renamed or
// individually-edited fixture that a single-file sha256 (the two
// above) can't.
// above) can't. The signed-durations fix (9865cc045aee) touches only
// the extractor and its unit test, not testdata/ci_analytics/, so this
// pin is unchanged and was re-verified against gascity's tree.
ciAnalyticsTestdataSHA256 = "dc9ac595df095ef8fe1b965d0e2a8d0b0e5694c3122208891d27655bcc2285d5"
)
@ -118,7 +120,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) {
}
sum := sha256.Sum256(raw)
if got := hex.EncodeToString(sum[:]); got != tc.want {
t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac): "+
t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extract-signed-durations @ 9865cc045aee1391dbcc7eac99339c26e3a5ce71): "+
"either this file drifted from gascity's canonical copy, or gascity changed it "+
"and this pin (and the vendored copy) needs updating to match", tc.path, got, tc.want)
}
@ -131,7 +133,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) {
}
}
// TestCIAnalyticsExtractUnitTests runs gascity's own 62 unittest cases
// TestCIAnalyticsExtractUnitTests runs gascity's own 67 unittest cases
// in-place against the vendored extractor: the real Bazel 9.2.0 fixtures
// (testdata/ci_analytics/real/*, regenerated under env -i after the S2
// leak), the synthetic remote/cache-hit exec log built by the test's own

View file

@ -60,7 +60,18 @@ import time
FORMAT = "ci-analytics-summary"
SCHEMA_VERSION = 1
EXTRACTOR_VERSION = "1.0.0"
# Bumped alongside the signed-duration fix below, so S4 can tell a
# fixed-up artifact (no more 2^63/2^64-scale decoy durations from a
# negative queue/exec/setup/upload time decoded as unsigned) from one
# produced by the earlier, buggy extractor.
EXTRACTOR_VERSION = "1.1.0"
# A per-spawn duration (Duration.seconds*1000 + Duration.nanos//1e6, after
# signed decoding) above this is treated as corrupt, not a real build: a
# single spawn cannot legitimately run for more than a day. Also the upper
# clamp for the other duration/int fields this file decodes (see
# _clamp_duration_ms below).
MAX_SANE_DURATION_MS = 24 * 60 * 60 * 1000
MAX_INVOCATIONS = 8
MAX_MNEMONICS = 200
@ -368,17 +379,56 @@ def fields(b, deadline=None):
yield f, t, v
def _to_signed64(v):
"""varint()/fields() decode a wire-type-0 field's raw bits as an
unsigned 64-bit value; protobuf's int64 (and int32 -- see
_to_signed32) are the two's-complement bit pattern of a *signed*
value, re-encoded with the same unsigned-varint algorithm. A real
negative int64 (e.g. a Duration whose seconds/nanos went negative from
clock skew between the NativeLink scheduler and a worker) is written
as a 10-byte varint of its 64-bit two's complement form, which this
recovers: values with the top bit set (>= 2^63) are negative."""
return v - (1 << 64) if v >= (1 << 63) else v
def _to_signed32(v):
"""Same, for an int32 field (Duration.nanos): proto encoders sign-
extend a negative int32 to 64 bits before varint-encoding it, so
_to_signed64(v) alone already recovers the right numeric value for a
well-formed input. This additionally truncates to the low 32 bits
first, so a corrupt/hostile encoder that puts a value outside the
int32 range on an int32-typed field degrades to *some* signed 32-bit
value instead of silently keeping 64 bits of it."""
v &= 0xFFFFFFFF
return v - (1 << 32) if v >= (1 << 31) else v
def _clamp_duration_ms(ms):
"""Clamps a decoded duration to [0, MAX_SANE_DURATION_MS]. A negative
duration -- the whole point of this fix: clock skew between the
NativeLink scheduler and a worker can make a remote spawn's queue time
read slightly negative, which _to_signed64() now decodes correctly
instead of as a near-2^63 positive decoy -- counts as 0, and so does
anything implausibly large (not a real build, so probably corrupt).
Applied per spawn/per field, before any sum: one poisoned spawn can
never poison a mnemonic's or label's total."""
if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0 or ms > MAX_SANE_DURATION_MS:
return 0
return ms
def dur_ms(v, deadline=None):
"""Decodes a Duration submessage's bytes (fields 1 seconds, 2 nanos)."""
"""Decodes a Duration submessage's bytes (fields 1 seconds, 2 nanos) to
a signed-then-clamped millisecond count."""
seconds = nanos = 0
for f, t, val in fields(v, deadline):
if t != 0:
continue
if f == 1:
seconds = val
seconds = _to_signed64(val)
elif f == 2:
nanos = val
return seconds * 1000 + nanos // 1_000_000
nanos = _to_signed32(val)
return _clamp_duration_ms(seconds * 1000 + nanos // 1_000_000)
def spawns(raw, deadline):
@ -425,7 +475,12 @@ def spawns(raw, deadline):
if mf in SPAWN_METRIC and mt == 2 and isinstance(mv, bytes):
s["m"][SPAWN_METRIC[mf]] = dur_ms(mv, deadline)
elif mf == 11 and mt == 0:
s["m"]["input_bytes"] = mv
# Also int64; not a duration, but still
# decoded as unsigned bits -- a negative
# value here is nonsensical, so clamp to 0
# rather than letting it decode to a huge
# positive "byte count" either.
s["m"]["input_bytes"] = max(0, _to_signed64(mv))
yield inv, s
@ -858,8 +913,14 @@ def build_action_data(events):
)
d["bep_executed"] += as_int(ad.get("actionsExecuted"))
d["bep_created"] += as_int(ad.get("actionsCreated"))
d["user_ms"] += as_duration_ms(ad.get("userTime")) or 0
d["system_ms"] += as_duration_ms(ad.get("systemTime")) or 0
# BEP's own Duration strings ("3.14s") aren't subject to the
# unsigned-varint bug (DURATION_RE allows a leading "-", so
# a legitimately negative one would already parse), but the
# same sane-duration clamp is applied here too, for the same
# reason: one corrupt/adversarial value should never poison
# this mnemonic's running total.
d["user_ms"] += _clamp_duration_ms(as_duration_ms(ad.get("userTime")) or 0)
d["system_ms"] += _clamp_duration_ms(as_duration_ms(ad.get("systemTime")) or 0)
except Exception:
continue
return data

View file

@ -79,6 +79,27 @@ def _duration_bytes(ms: int) -> bytes:
return _varint_field(1, seconds) + _varint_field(2, nanos * 1_000_000)
def _signed_varint(n: int) -> bytes:
"""Encodes a signed int64/int32 field's value the way a real protobuf
encoder does: the 64-bit two's-complement bit pattern, varint-encoded
as if it were unsigned -- a negative value always takes the full 10
bytes. _varint() itself can't be used directly for a negative n: `n &
0x7F`/`n >>= 7` on a negative Python int never reaches 0 (arithmetic
shift preserves the sign), so it would loop forever."""
return _varint(n & 0xFFFFFFFFFFFFFFFF)
def _signed_varint_field(field: int, n: int) -> bytes:
return _tag(field, 0) + _signed_varint(n)
def _signed_duration_bytes(seconds: int, nanos: int) -> bytes:
"""Builds a Duration submessage (fields 1 seconds, 2 nanos) with
explicit, possibly-negative values, each encoded the way a real
negative int64/int32 would be: a 10-byte two's-complement varint."""
return _signed_varint_field(1, seconds) + _signed_varint_field(2, nanos)
def _spawn_metrics(total_ms=0, queue_ms=0, exec_ms=0, input_bytes=0) -> bytes:
body = b""
if total_ms:
@ -293,6 +314,31 @@ class ExtractorHelperTests(unittest.TestCase):
self.assertEqual(extract._clamp_pct(float("-inf")), 0.0)
self.assertEqual(extract._clamp_pct(float("nan")), 0.0)
def test_to_signed64_recovers_negative_values_from_the_raw_varint_bits(self):
# -1 as a 64-bit two's complement varint is 10 bytes of 0xFF-ish
# bits; varint() decodes that as 2**64 - 1.
self.assertEqual(extract._to_signed64((1 << 64) - 1), -1)
self.assertEqual(extract._to_signed64(1 << 63), -(1 << 63))
self.assertEqual(extract._to_signed64(0), 0)
self.assertEqual(extract._to_signed64((1 << 63) - 1), (1 << 63) - 1)
def test_to_signed32_truncates_then_sign_extends(self):
self.assertEqual(extract._to_signed32((1 << 32) - 1), -1)
self.assertEqual(extract._to_signed32(1 << 31), -(1 << 31))
self.assertEqual(extract._to_signed32(0), 0)
# A hostile/corrupt int64-range value on an int32-typed field is
# truncated to the low 32 bits first, not kept as a huge int64.
self.assertEqual(extract._to_signed32((1 << 40) + 5), 5)
def test_clamp_duration_ms_rejects_negative_and_absurdly_large(self):
self.assertEqual(extract._clamp_duration_ms(500), 500)
self.assertEqual(extract._clamp_duration_ms(0), 0)
self.assertEqual(extract._clamp_duration_ms(-1500), 0)
self.assertEqual(extract._clamp_duration_ms(extract.MAX_SANE_DURATION_MS), extract.MAX_SANE_DURATION_MS)
self.assertEqual(extract._clamp_duration_ms(extract.MAX_SANE_DURATION_MS + 1), 0)
self.assertEqual(extract._clamp_duration_ms(float("inf")), 0)
self.assertEqual(extract._clamp_duration_ms(float("nan")), 0)
class RealFixtureTests(unittest.TestCase):
"""testdata/ci_analytics/real/*: genuine Bazel 9.2.0 output."""
@ -1161,6 +1207,63 @@ class RobustnessTests(unittest.TestCase):
self.assertEqual(status, "corrupt")
self.assertEqual([r["repo"] for r in rows], ["@@rules_cc+"])
def test_negative_queue_duration_decodes_to_zero_not_a_huge_positive(self):
"""The production bug: clock skew between the NativeLink scheduler
and a worker can make a remote spawn's queue time read slightly
negative. protobuf encodes that as a 10-byte two's-complement
varint of the Duration's seconds/nanos; decoding those bits as
unsigned (the pre-fix behavior) produced "remote_queue_ms" values
around 2**63/1e3 -- not a small negative number. Built with the
test-side signed varint encoder (_signed_duration_bytes), not the
plain _duration_bytes() helper, which can't represent a negative
value at all."""
uuid = "11111111-2222-3333-4444-555555555555"
# -1.5s of (impossible, but exactly what clock skew would produce)
# negative queue time, alongside a normal positive total/exec so
# the fix's "other values intact" half is also checked.
metrics = (
_bytes_field(5, _signed_duration_bytes(-1, -500_000_000))
+ _bytes_field(1, _duration_bytes(200))
+ _bytes_field(8, _duration_bytes(50))
)
spawn_body = _spawn("//:synth_remote2", "TestRunner", "remote", False, metrics)
exec_log = _invocation_entry(uuid) + _entry_with_spawn(spawn_body)
rec = self._run(self._bep(uuid), exec_log)
self.assertIsNotNone(rec)
self.assertEqual(rec["exec_log"], "ok")
by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]}
tr = by_mnemonic["TestRunner"]
self.assertEqual(tr["remote_queue_ms"], 0)
self.assertEqual(tr["remote_queue_max_ms"], 0)
# Not poisoned by the clamped-away queue value: total/exec, decoded
# from the same spawn's other, non-negative Duration fields, are
# unaffected.
self.assertEqual(tr["total_ms"], 200)
self.assertEqual(tr["remote_exec_ms"], 50)
def test_absurdly_large_duration_clamps_to_zero(self):
"""A corrupt (not just adversarial) producer could put an
implausible but individually well-formed value on a Duration
field -- MAX_SANE_DURATION_MS guards against that the same way as
the negative case, so one bad spawn can't blow out a mnemonic's
sum in either direction."""
uuid = "11111111-2222-3333-4444-555555555555"
absurd_ms = 50 * 60 * 60 * 1000 # 50 hours: no real spawn runs this long
metrics = (
_bytes_field(5, _duration_bytes(absurd_ms))
+ _bytes_field(1, _duration_bytes(200))
+ _bytes_field(8, _duration_bytes(50))
)
spawn_body = _spawn("//:synth_remote3", "TestRunner", "remote", False, metrics)
exec_log = _invocation_entry(uuid) + _entry_with_spawn(spawn_body)
rec = self._run(self._bep(uuid), exec_log)
self.assertIsNotNone(rec)
by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]}
tr = by_mnemonic["TestRunner"]
self.assertEqual(tr["remote_queue_ms"], 0)
self.assertEqual(tr["total_ms"], 200)
self.assertEqual(tr["remote_exec_ms"], 50)
class BoundsTests(unittest.TestCase):
def test_exec_log_parse_timeout_is_reported(self):