beads/scripts/ci_analytics_extract_test.go
Julian Knutsen a945619b5e
fix(bazel): re-vendor the BEP test-timing clamp from gascity (extractor 1.1.1) (#7477)
#7374 re-vendored gascity's signed-duration fix for exec-log spawns, but
build_tests() still kept the sign of BEP testResult timings. rbe-west
scheduler/worker clock skew makes a remotely executed test report
queueTime -1 ms (-24 ms at worst), and the collector rejected the whole
artifact on tests[].results[].queue_ms: about 19% of beads artifacts,
holding 52% of beads' CI worker time.

Byte copy of gascity 2ddde0d94be15d95086cc2bcb169303ea1243456: clamp
duration_ms and every timingBreakdown child with _clamp_duration_ms.
Updates the two sha256 pins; testdata/ci_analytics is unchanged.

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-10 16:20:37 +00:00

158 lines
6.7 KiB
Go

package scripts_test
import (
"crypto/sha256"
"encoding/hex"
"io/fs"
"os"
"os/exec"
"path/filepath"
"sort"
"testing"
)
// ciAnalyticsExtractScript and ciAnalyticsExtractUnitTest are a byte copy of
// gascity's tools/bazel/ci_analytics_extract.py and
// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extract-signed-durations,
// gascity commit 9865cc045aee1391dbcc7eac99339c26e3a5ce71, a production bug
// fix on top of the final squashed-to-one-commit S2 history at
// 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac: negative protobuf durations were
// decoded as unsigned, so the collector rejected roughly 22% of artifacts on
// remote_queue_ms. Re-vendored at gascity commit
// 2ddde0d94be15d95086cc2bcb169303ea1243456
// (fix/ci-analytics-clamp-test-timings, extractor 1.1.1): BEP test timings
// kept their sign, so a remote test's -1 ms queueTime (clock skew) still
// rejected 17-32% of artifacts on tests[].results[].queue_ms. The fixtures
// are generated under env -i and committed
// with the compact exec log decompressed (*.binpb), never a zstd frame, so
// nothing opaque reaches version control, the same way
// tools/rbe/cache-zstd-probe.sh is a byte copy. gascity is canonical for
// this pair; TestCIAnalyticsExtractorIsPinnedToGascity below pins both
// files' sha256 so a local edit here is caught instead of silently
// drifting out of lockstep. The vendored test file's own 70 cases (run in
// place by TestCIAnalyticsExtractUnitTests) now include the
// mismatched-invocation-id and truncated-BEP cases beads originally added
// (test_beads_mismatch_fixture_reports_mismatch,
// test_truncated_last_line_sets_bep_truncated), so no separate Go-side
// duplicate of either remains here.
func ciAnalyticsExtractScript(root string) string {
return filepath.Join(root, "tools", "bazel", "ci_analytics_extract.py")
}
func ciAnalyticsExtractUnitTest(root string) string {
return filepath.Join(root, "tools", "bazel", "ci_analytics_extract_test.py")
}
func ciAnalyticsTestdataDir(root string) string {
return filepath.Join(root, "tools", "bazel", "testdata", "ci_analytics")
}
// ciAnalyticsExtractSHA256 and ciAnalyticsExtractUnitTestSHA256 pin the
// vendored files' content, computed from gascity's test-timing clamp at
// 2ddde0d94be15d95086cc2bcb169303ea1243456 (extractor 1.1.1).
const (
ciAnalyticsExtractSHA256 = "1d0049bc37d957a73c6c6f808d6103ddad290a8dd053980ec9f97d4750bed2b7"
ciAnalyticsExtractUnitTestSHA256 = "42331bf3e5ce5e650f62f7c00913b0155416352360e3b1d99be58e5508d22f9f"
// ciAnalyticsTestdataSHA256 pins testdata/ci_analytics/** as a whole
// (regen.sh, mismatch.bep.jsonl, poisoned-bep.json and real/*): a sha256
// over every file's slash-joined relative path and bytes, each
// NUL-terminated, sorted by path (ciAnalyticsTreeSHA256 below) -
// computed the same way over gascity's 00161373bc tree and confirmed
// identical, so this one hash catches a dropped, renamed or
// individually-edited fixture that a single-file sha256 (the two
// above) can't. The signed-durations fix (9865cc045aee) touches only
// the extractor and its unit test, not testdata/ci_analytics/, so this
// pin is unchanged and was re-verified against gascity's tree. The
// test-timing clamp (2ddde0d94be1) likewise leaves testdata alone.
ciAnalyticsTestdataSHA256 = "dc9ac595df095ef8fe1b965d0e2a8d0b0e5694c3122208891d27655bcc2285d5"
)
// ciAnalyticsTreeSHA256 hashes every regular file under dir: each entry
// contributes its dir-relative, slash-joined path, a NUL, its bytes and a
// NUL, in path-sorted order, so the result depends on the exact set of
// files present (not just their union of bytes) and is independent of the
// OS's directory-walk order.
func ciAnalyticsTreeSHA256(t *testing.T, dir string) string {
t.Helper()
var paths []string
if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
rel, err := filepath.Rel(dir, path)
if err != nil {
return err
}
paths = append(paths, filepath.ToSlash(rel))
return nil
}); err != nil {
t.Fatalf("walking %s: %v", dir, err)
}
sort.Strings(paths)
h := sha256.New()
for _, rel := range paths {
raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel)))
if err != nil {
t.Fatalf("reading %s: %v", rel, err)
}
h.Write([]byte(rel))
h.Write([]byte{0})
h.Write(raw)
h.Write([]byte{0})
}
return hex.EncodeToString(h.Sum(nil))
}
// TestCIAnalyticsExtractorIsPinnedToGascity fails the moment either vendored
// file is edited locally without also updating the pin (and, ideally,
// upstreaming the change to gascity first): the design treats gascity as
// canonical for this pair, like cache-zstd-probe.sh.
func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) {
root := sourceRepoRoot(t)
for _, tc := range []struct {
path string
want string
}{
{ciAnalyticsExtractScript(root), ciAnalyticsExtractSHA256},
{ciAnalyticsExtractUnitTest(root), ciAnalyticsExtractUnitTestSHA256},
} {
raw, err := os.ReadFile(tc.path)
if err != nil {
t.Fatalf("reading %s: %v", tc.path, err)
}
sum := sha256.Sum256(raw)
if got := hex.EncodeToString(sum[:]); got != tc.want {
t.Errorf("%s sha256 = %s, want %s (gascity fix/ci-analytics-clamp-test-timings @ 2ddde0d94be15d95086cc2bcb169303ea1243456, extractor 1.1.1): "+
"either this file drifted from gascity's canonical copy, or gascity changed it "+
"and this pin (and the vendored copy) needs updating to match", tc.path, got, tc.want)
}
}
testdataDir := ciAnalyticsTestdataDir(root)
if got := ciAnalyticsTreeSHA256(t, testdataDir); got != ciAnalyticsTestdataSHA256 {
t.Errorf("%s tree sha256 = %s, want %s (gascity ci/analytics-extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac): "+
"a fixture under testdata/ci_analytics/ was added, removed, renamed or edited without updating this pin", testdataDir, got, ciAnalyticsTestdataSHA256)
}
}
// TestCIAnalyticsExtractUnitTests runs gascity's own 70 unittest cases
// in-place against the vendored extractor: the real Bazel 9.2.0 fixtures
// (testdata/ci_analytics/real/*, regenerated under env -i after the S2
// leak), the synthetic remote/cache-hit exec log built by the test's own
// varint encoder, the poisoned/malformed BEP and exec-log robustness cases,
// and the ported mismatched-invocation-id and truncated-BEP cases. This is
// the "thin Go wrapper" around the vendored pair.
func TestCIAnalyticsExtractUnitTests(t *testing.T) {
root := sourceRepoRoot(t)
python := requireHostTool(t, "python3")
cmd := exec.Command(python, ciAnalyticsExtractUnitTest(root), "-v")
cmd.Dir = root
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("ci_analytics_extract_test.py failed: %v\n%s", err, out)
}
}