Compare commits
1 commit
main
...
ci/analyti
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
df72811377 |
3 changed files with 190 additions and 24 deletions
|
|
@ -13,18 +13,18 @@ import (
|
|||
|
||||
// ciAnalyticsExtractScript and ciAnalyticsExtractUnitTest are a byte copy of
|
||||
// gascity's tools/bazel/ci_analytics_extract.py and
|
||||
// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extractor,
|
||||
// gascity commit 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac, the final
|
||||
// squashed-to-one-commit history superseding 32d51ce26a and the original
|
||||
// 9104311a8a after the S2 review found the first cut's real/*-bep.json
|
||||
// fixtures leaked operator env: a token, SSH IPs, the OpenBao address, the
|
||||
// hostname and home paths. The fixtures are generated under env -i and
|
||||
// committed with the compact exec log decompressed (*.binpb), never a
|
||||
// zstd frame, so nothing opaque reaches version control), the same way
|
||||
// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extract-signed-durations,
|
||||
// gascity commit 9865cc045aee1391dbcc7eac99339c26e3a5ce71, a production bug
|
||||
// fix on top of the final squashed-to-one-commit S2 history at
|
||||
// 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac: negative protobuf durations were
|
||||
// decoded as unsigned, so the collector rejected roughly 22% of artifacts on
|
||||
// remote_queue_ms. The fixtures are generated under env -i and committed
|
||||
// with the compact exec log decompressed (*.binpb), never a zstd frame, so
|
||||
// nothing opaque reaches version control, the same way
|
||||
// tools/rbe/cache-zstd-probe.sh is a byte copy. gascity is canonical for
|
||||
// this pair; TestCIAnalyticsExtractorIsPinnedToGascity below pins both
|
||||
// files' sha256 so a local edit here is caught instead of silently
|
||||
// drifting out of lockstep. The vendored test file's own 62 cases (run in
|
||||
// drifting out of lockstep. The vendored test file's own 67 cases (run in
|
||||
// place by TestCIAnalyticsExtractUnitTests) now include the
|
||||
// mismatched-invocation-id and truncated-BEP cases beads originally added
|
||||
// (test_beads_mismatch_fixture_reports_mismatch,
|
||||
|
|
@ -43,11 +43,11 @@ func ciAnalyticsTestdataDir(root string) string {
|
|||
}
|
||||
|
||||
// ciAnalyticsExtractSHA256 and ciAnalyticsExtractUnitTestSHA256 pin the
|
||||
// vendored files' content, computed from gascity's final sanitized,
|
||||
// squashed-to-one-commit history at 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac.
|
||||
// vendored files' content, computed from gascity's signed-durations fix at
|
||||
// 9865cc045aee1391dbcc7eac99339c26e3a5ce71.
|
||||
const (
|
||||
ciAnalyticsExtractSHA256 = "6bb912b17faac59a12ec53f71fbfbedfecc503037b6f5954e878d108abaae44f"
|
||||
ciAnalyticsExtractUnitTestSHA256 = "e833e1c961010cd02189afaef273a261a54ccabe916b03bae3115177378a405b"
|
||||
ciAnalyticsExtractSHA256 = "be6d187804f8b44a75af40113fef97c37476923a1c6a0966d6216a791935c674"
|
||||
ciAnalyticsExtractUnitTestSHA256 = "7e67b54b0e6d374efed899c8cae448f75effcdd1210b053e242a3ff51f1c8992"
|
||||
// ciAnalyticsTestdataSHA256 pins testdata/ci_analytics/** as a whole
|
||||
// (regen.sh, mismatch.bep.jsonl, poisoned-bep.json and real/*): a sha256
|
||||
// over every file's slash-joined relative path and bytes, each
|
||||
|
|
@ -55,7 +55,9 @@ const (
|
|||
// computed the same way over gascity's 00161373bc tree and confirmed
|
||||
// identical, so this one hash catches a dropped, renamed or
|
||||
// individually-edited fixture that a single-file sha256 (the two
|
||||
// above) can't.
|
||||
// above) can't. The signed-durations fix (9865cc045aee) touches only
|
||||
// the extractor and its unit test, not testdata/ci_analytics/, so this
|
||||
// pin is unchanged and was re-verified against gascity's tree.
|
||||
ciAnalyticsTestdataSHA256 = "dc9ac595df095ef8fe1b965d0e2a8d0b0e5694c3122208891d27655bcc2285d5"
|
||||
)
|
||||
|
||||
|
|
@ -118,7 +120,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) {
|
|||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
if got := hex.EncodeToString(sum[:]); got != tc.want {
|
||||
t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac): "+
|
||||
t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extract-signed-durations @ 9865cc045aee1391dbcc7eac99339c26e3a5ce71): "+
|
||||
"either this file drifted from gascity's canonical copy, or gascity changed it "+
|
||||
"and this pin (and the vendored copy) needs updating to match", tc.path, got, tc.want)
|
||||
}
|
||||
|
|
@ -131,7 +133,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestCIAnalyticsExtractUnitTests runs gascity's own 62 unittest cases
|
||||
// TestCIAnalyticsExtractUnitTests runs gascity's own 67 unittest cases
|
||||
// in-place against the vendored extractor: the real Bazel 9.2.0 fixtures
|
||||
// (testdata/ci_analytics/real/*, regenerated under env -i after the S2
|
||||
// leak), the synthetic remote/cache-hit exec log built by the test's own
|
||||
|
|
|
|||
|
|
@ -60,7 +60,18 @@ import time
|
|||
|
||||
FORMAT = "ci-analytics-summary"
|
||||
SCHEMA_VERSION = 1
|
||||
EXTRACTOR_VERSION = "1.0.0"
|
||||
# Bumped alongside the signed-duration fix below, so S4 can tell a
|
||||
# fixed-up artifact (no more 2^63/2^64-scale decoy durations from a
|
||||
# negative queue/exec/setup/upload time decoded as unsigned) from one
|
||||
# produced by the earlier, buggy extractor.
|
||||
EXTRACTOR_VERSION = "1.1.0"
|
||||
|
||||
# A per-spawn duration (Duration.seconds*1000 + Duration.nanos//1e6, after
|
||||
# signed decoding) above this is treated as corrupt, not a real build: a
|
||||
# single spawn cannot legitimately run for more than a day. Also the upper
|
||||
# clamp for the other duration/int fields this file decodes (see
|
||||
# _clamp_duration_ms below).
|
||||
MAX_SANE_DURATION_MS = 24 * 60 * 60 * 1000
|
||||
|
||||
MAX_INVOCATIONS = 8
|
||||
MAX_MNEMONICS = 200
|
||||
|
|
@ -368,17 +379,56 @@ def fields(b, deadline=None):
|
|||
yield f, t, v
|
||||
|
||||
|
||||
def _to_signed64(v):
|
||||
"""varint()/fields() decode a wire-type-0 field's raw bits as an
|
||||
unsigned 64-bit value; protobuf's int64 (and int32 -- see
|
||||
_to_signed32) are the two's-complement bit pattern of a *signed*
|
||||
value, re-encoded with the same unsigned-varint algorithm. A real
|
||||
negative int64 (e.g. a Duration whose seconds/nanos went negative from
|
||||
clock skew between the NativeLink scheduler and a worker) is written
|
||||
as a 10-byte varint of its 64-bit two's complement form, which this
|
||||
recovers: values with the top bit set (>= 2^63) are negative."""
|
||||
return v - (1 << 64) if v >= (1 << 63) else v
|
||||
|
||||
|
||||
def _to_signed32(v):
|
||||
"""Same, for an int32 field (Duration.nanos): proto encoders sign-
|
||||
extend a negative int32 to 64 bits before varint-encoding it, so
|
||||
_to_signed64(v) alone already recovers the right numeric value for a
|
||||
well-formed input. This additionally truncates to the low 32 bits
|
||||
first, so a corrupt/hostile encoder that puts a value outside the
|
||||
int32 range on an int32-typed field degrades to *some* signed 32-bit
|
||||
value instead of silently keeping 64 bits of it."""
|
||||
v &= 0xFFFFFFFF
|
||||
return v - (1 << 32) if v >= (1 << 31) else v
|
||||
|
||||
|
||||
def _clamp_duration_ms(ms):
|
||||
"""Clamps a decoded duration to [0, MAX_SANE_DURATION_MS]. A negative
|
||||
duration -- the whole point of this fix: clock skew between the
|
||||
NativeLink scheduler and a worker can make a remote spawn's queue time
|
||||
read slightly negative, which _to_signed64() now decodes correctly
|
||||
instead of as a near-2^63 positive decoy -- counts as 0, and so does
|
||||
anything implausibly large (not a real build, so probably corrupt).
|
||||
Applied per spawn/per field, before any sum: one poisoned spawn can
|
||||
never poison a mnemonic's or label's total."""
|
||||
if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0 or ms > MAX_SANE_DURATION_MS:
|
||||
return 0
|
||||
return ms
|
||||
|
||||
|
||||
def dur_ms(v, deadline=None):
|
||||
"""Decodes a Duration submessage's bytes (fields 1 seconds, 2 nanos)."""
|
||||
"""Decodes a Duration submessage's bytes (fields 1 seconds, 2 nanos) to
|
||||
a signed-then-clamped millisecond count."""
|
||||
seconds = nanos = 0
|
||||
for f, t, val in fields(v, deadline):
|
||||
if t != 0:
|
||||
continue
|
||||
if f == 1:
|
||||
seconds = val
|
||||
seconds = _to_signed64(val)
|
||||
elif f == 2:
|
||||
nanos = val
|
||||
return seconds * 1000 + nanos // 1_000_000
|
||||
nanos = _to_signed32(val)
|
||||
return _clamp_duration_ms(seconds * 1000 + nanos // 1_000_000)
|
||||
|
||||
|
||||
def spawns(raw, deadline):
|
||||
|
|
@ -425,7 +475,12 @@ def spawns(raw, deadline):
|
|||
if mf in SPAWN_METRIC and mt == 2 and isinstance(mv, bytes):
|
||||
s["m"][SPAWN_METRIC[mf]] = dur_ms(mv, deadline)
|
||||
elif mf == 11 and mt == 0:
|
||||
s["m"]["input_bytes"] = mv
|
||||
# Also int64; not a duration, but still
|
||||
# decoded as unsigned bits -- a negative
|
||||
# value here is nonsensical, so clamp to 0
|
||||
# rather than letting it decode to a huge
|
||||
# positive "byte count" either.
|
||||
s["m"]["input_bytes"] = max(0, _to_signed64(mv))
|
||||
yield inv, s
|
||||
|
||||
|
||||
|
|
@ -858,8 +913,14 @@ def build_action_data(events):
|
|||
)
|
||||
d["bep_executed"] += as_int(ad.get("actionsExecuted"))
|
||||
d["bep_created"] += as_int(ad.get("actionsCreated"))
|
||||
d["user_ms"] += as_duration_ms(ad.get("userTime")) or 0
|
||||
d["system_ms"] += as_duration_ms(ad.get("systemTime")) or 0
|
||||
# BEP's own Duration strings ("3.14s") aren't subject to the
|
||||
# unsigned-varint bug (DURATION_RE allows a leading "-", so
|
||||
# a legitimately negative one would already parse), but the
|
||||
# same sane-duration clamp is applied here too, for the same
|
||||
# reason: one corrupt/adversarial value should never poison
|
||||
# this mnemonic's running total.
|
||||
d["user_ms"] += _clamp_duration_ms(as_duration_ms(ad.get("userTime")) or 0)
|
||||
d["system_ms"] += _clamp_duration_ms(as_duration_ms(ad.get("systemTime")) or 0)
|
||||
except Exception:
|
||||
continue
|
||||
return data
|
||||
|
|
|
|||
|
|
@ -79,6 +79,27 @@ def _duration_bytes(ms: int) -> bytes:
|
|||
return _varint_field(1, seconds) + _varint_field(2, nanos * 1_000_000)
|
||||
|
||||
|
||||
def _signed_varint(n: int) -> bytes:
|
||||
"""Encodes a signed int64/int32 field's value the way a real protobuf
|
||||
encoder does: the 64-bit two's-complement bit pattern, varint-encoded
|
||||
as if it were unsigned -- a negative value always takes the full 10
|
||||
bytes. _varint() itself can't be used directly for a negative n: `n &
|
||||
0x7F`/`n >>= 7` on a negative Python int never reaches 0 (arithmetic
|
||||
shift preserves the sign), so it would loop forever."""
|
||||
return _varint(n & 0xFFFFFFFFFFFFFFFF)
|
||||
|
||||
|
||||
def _signed_varint_field(field: int, n: int) -> bytes:
|
||||
return _tag(field, 0) + _signed_varint(n)
|
||||
|
||||
|
||||
def _signed_duration_bytes(seconds: int, nanos: int) -> bytes:
|
||||
"""Builds a Duration submessage (fields 1 seconds, 2 nanos) with
|
||||
explicit, possibly-negative values, each encoded the way a real
|
||||
negative int64/int32 would be: a 10-byte two's-complement varint."""
|
||||
return _signed_varint_field(1, seconds) + _signed_varint_field(2, nanos)
|
||||
|
||||
|
||||
def _spawn_metrics(total_ms=0, queue_ms=0, exec_ms=0, input_bytes=0) -> bytes:
|
||||
body = b""
|
||||
if total_ms:
|
||||
|
|
@ -293,6 +314,31 @@ class ExtractorHelperTests(unittest.TestCase):
|
|||
self.assertEqual(extract._clamp_pct(float("-inf")), 0.0)
|
||||
self.assertEqual(extract._clamp_pct(float("nan")), 0.0)
|
||||
|
||||
def test_to_signed64_recovers_negative_values_from_the_raw_varint_bits(self):
|
||||
# -1 as a 64-bit two's complement varint is 10 bytes of 0xFF-ish
|
||||
# bits; varint() decodes that as 2**64 - 1.
|
||||
self.assertEqual(extract._to_signed64((1 << 64) - 1), -1)
|
||||
self.assertEqual(extract._to_signed64(1 << 63), -(1 << 63))
|
||||
self.assertEqual(extract._to_signed64(0), 0)
|
||||
self.assertEqual(extract._to_signed64((1 << 63) - 1), (1 << 63) - 1)
|
||||
|
||||
def test_to_signed32_truncates_then_sign_extends(self):
|
||||
self.assertEqual(extract._to_signed32((1 << 32) - 1), -1)
|
||||
self.assertEqual(extract._to_signed32(1 << 31), -(1 << 31))
|
||||
self.assertEqual(extract._to_signed32(0), 0)
|
||||
# A hostile/corrupt int64-range value on an int32-typed field is
|
||||
# truncated to the low 32 bits first, not kept as a huge int64.
|
||||
self.assertEqual(extract._to_signed32((1 << 40) + 5), 5)
|
||||
|
||||
def test_clamp_duration_ms_rejects_negative_and_absurdly_large(self):
|
||||
self.assertEqual(extract._clamp_duration_ms(500), 500)
|
||||
self.assertEqual(extract._clamp_duration_ms(0), 0)
|
||||
self.assertEqual(extract._clamp_duration_ms(-1500), 0)
|
||||
self.assertEqual(extract._clamp_duration_ms(extract.MAX_SANE_DURATION_MS), extract.MAX_SANE_DURATION_MS)
|
||||
self.assertEqual(extract._clamp_duration_ms(extract.MAX_SANE_DURATION_MS + 1), 0)
|
||||
self.assertEqual(extract._clamp_duration_ms(float("inf")), 0)
|
||||
self.assertEqual(extract._clamp_duration_ms(float("nan")), 0)
|
||||
|
||||
|
||||
class RealFixtureTests(unittest.TestCase):
|
||||
"""testdata/ci_analytics/real/*: genuine Bazel 9.2.0 output."""
|
||||
|
|
@ -1161,6 +1207,63 @@ class RobustnessTests(unittest.TestCase):
|
|||
self.assertEqual(status, "corrupt")
|
||||
self.assertEqual([r["repo"] for r in rows], ["@@rules_cc+"])
|
||||
|
||||
def test_negative_queue_duration_decodes_to_zero_not_a_huge_positive(self):
|
||||
"""The production bug: clock skew between the NativeLink scheduler
|
||||
and a worker can make a remote spawn's queue time read slightly
|
||||
negative. protobuf encodes that as a 10-byte two's-complement
|
||||
varint of the Duration's seconds/nanos; decoding those bits as
|
||||
unsigned (the pre-fix behavior) produced "remote_queue_ms" values
|
||||
around 2**63/1e3 -- not a small negative number. Built with the
|
||||
test-side signed varint encoder (_signed_duration_bytes), not the
|
||||
plain _duration_bytes() helper, which can't represent a negative
|
||||
value at all."""
|
||||
uuid = "11111111-2222-3333-4444-555555555555"
|
||||
# -1.5s of (impossible, but exactly what clock skew would produce)
|
||||
# negative queue time, alongside a normal positive total/exec so
|
||||
# the fix's "other values intact" half is also checked.
|
||||
metrics = (
|
||||
_bytes_field(5, _signed_duration_bytes(-1, -500_000_000))
|
||||
+ _bytes_field(1, _duration_bytes(200))
|
||||
+ _bytes_field(8, _duration_bytes(50))
|
||||
)
|
||||
spawn_body = _spawn("//:synth_remote2", "TestRunner", "remote", False, metrics)
|
||||
exec_log = _invocation_entry(uuid) + _entry_with_spawn(spawn_body)
|
||||
rec = self._run(self._bep(uuid), exec_log)
|
||||
self.assertIsNotNone(rec)
|
||||
self.assertEqual(rec["exec_log"], "ok")
|
||||
by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]}
|
||||
tr = by_mnemonic["TestRunner"]
|
||||
self.assertEqual(tr["remote_queue_ms"], 0)
|
||||
self.assertEqual(tr["remote_queue_max_ms"], 0)
|
||||
# Not poisoned by the clamped-away queue value: total/exec, decoded
|
||||
# from the same spawn's other, non-negative Duration fields, are
|
||||
# unaffected.
|
||||
self.assertEqual(tr["total_ms"], 200)
|
||||
self.assertEqual(tr["remote_exec_ms"], 50)
|
||||
|
||||
def test_absurdly_large_duration_clamps_to_zero(self):
|
||||
"""A corrupt (not just adversarial) producer could put an
|
||||
implausible but individually well-formed value on a Duration
|
||||
field -- MAX_SANE_DURATION_MS guards against that the same way as
|
||||
the negative case, so one bad spawn can't blow out a mnemonic's
|
||||
sum in either direction."""
|
||||
uuid = "11111111-2222-3333-4444-555555555555"
|
||||
absurd_ms = 50 * 60 * 60 * 1000 # 50 hours: no real spawn runs this long
|
||||
metrics = (
|
||||
_bytes_field(5, _duration_bytes(absurd_ms))
|
||||
+ _bytes_field(1, _duration_bytes(200))
|
||||
+ _bytes_field(8, _duration_bytes(50))
|
||||
)
|
||||
spawn_body = _spawn("//:synth_remote3", "TestRunner", "remote", False, metrics)
|
||||
exec_log = _invocation_entry(uuid) + _entry_with_spawn(spawn_body)
|
||||
rec = self._run(self._bep(uuid), exec_log)
|
||||
self.assertIsNotNone(rec)
|
||||
by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]}
|
||||
tr = by_mnemonic["TestRunner"]
|
||||
self.assertEqual(tr["remote_queue_ms"], 0)
|
||||
self.assertEqual(tr["total_ms"], 200)
|
||||
self.assertEqual(tr["remote_exec_ms"], 50)
|
||||
|
||||
|
||||
class BoundsTests(unittest.TestCase):
|
||||
def test_exec_log_parse_timeout_is_reported(self):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue