tools/bazel/ci_analytics_extract.py turns a lane's BEP file, compact
execution log and JSON profile into a redacted ci-analytics-summary
(timing, per-mnemonic cache classes, cold actions, test results,
repo-fetch time): only validated labels, mnemonics, enums and numbers
leave the runner, never an endpoint, cert/key path, env value, token
or hostname. Every lane's "CI analytics summary" step runs the
extractor and uploads its output as ci-analytics-<lane>-<run_attempt>
(7-day retention); CHECK_RUN_ID uses the real job.check_run_id
context (requires actionlint >= 1.7.8), and both the summary and
upload steps are continue-on-error, the summary capped at
timeout-minutes: 3, and both gated on !cancelled() (not always(),
which also runs after a workflow cancellation).
The extractor, its 62-case unittest file, and testdata/ci_analytics/
(the real/* Bazel 9.2.0 fixtures, poisoned-bep.json, mismatch.bep.jsonl
and regen.sh) are vendored byte-identical from gascity's
ci/analytics-extractor slice (final squashed commit
00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac), which is canonical for this
pair the same way tools/rbe/cache-zstd-probe.sh is a byte copy.
Fixtures are regenerated under env -i with every BEP field stripped
unless it's on an explicit allowlist, and the compact exec log is
committed decompressed (*.binpb) rather than as a zstd frame, so
nothing opaque reaches version control.
scripts/ci_analytics_extract_test.go is the thin Go wrapper:
TestCIAnalyticsExtractorIsPinnedToGascity pins both vendored files'
sha256 so local drift is caught, and TestCIAnalyticsExtractUnitTests
runs gascity's unittest file in place. scripts/ci_analytics_workflow_test.go's
TestCIAnalyticsStepsAreSafeAndBounded mirrors gascity's pin test,
adapted to beads' 8 separately-named lane jobs: continue-on-error,
timeout-minutes bounds, no raw ${{ }} expression in run:, !cancelled()
required in if:, correct per-lane upload artifact name/path, and
CHECK_RUN_ID == ${{ job.check_run_id }}, for every lane.
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
282 lines
13 KiB
Go
282 lines
13 KiB
Go
package scripts_test
|
|
|
|
import (
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// ciAnalyticsLaneJobs lists bazel.yml's lane jobs that carry the "CI
|
|
// analytics summary"/"Upload CI analytics summary" steps
|
|
// (rbe-ci-bep-analytics-design.md, S3), mapped to each job's LANE env
|
|
// value. The artifact name and the LANE env are both derived from this one
|
|
// map in the checks below, instead of being pinned independently and
|
|
// allowed to drift apart.
|
|
var ciAnalyticsLaneJobs = map[string]string{
|
|
"bazel-test": "test",
|
|
"bazel-pure": "pure",
|
|
"bazel-embedded": "embedded",
|
|
"bazel-integration": "integration",
|
|
"bazel-doltserver": "doltserver",
|
|
"bazel-proxied": "proxied-server",
|
|
"bazel-server-storage": "server-storage",
|
|
"bazel-cmd-dolt": "cmd-dolt",
|
|
}
|
|
|
|
// ciAnalyticsExcludedJobs are bazel.yml jobs that deliberately do not run
|
|
// the CI analytics extractor, with the reason: mirrors
|
|
// TestBazelLaneIsGatedAlongsideLegacy's gated/advisory/flag-gated split, so
|
|
// a new job in bazel.yml can't silently go unaccounted for here either.
|
|
var ciAnalyticsExcludedJobs = map[string]string{
|
|
"rbe": "decides the execution mode; builds nothing itself, so there is no BEP/exec-log/profile to summarize",
|
|
"rbe-prewarm": "best-effort pre-warm dispatch to gascity's rbe-west pool; not a build/test lane",
|
|
"package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract",
|
|
"package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract",
|
|
}
|
|
|
|
// ciAnalyticsRunTemplateRE finds a GitHub Actions expression (${{ ... }})
|
|
// inside a `run:` block: ci_analytics_extract.py's arguments must come from
|
|
// $LANE/$MODE/$CHECK_RUN_ID/$PR_HINT shell variables (set in `env:`, which
|
|
// GitHub Actions substitutes before the shell ever sees them), never a raw
|
|
// ${{ }} expression spliced into the command line itself.
|
|
var ciAnalyticsRunTemplateRE = regexp.MustCompile(`\$\{\{`)
|
|
|
|
// ciAnalyticsRunnerTempPathRE captures the basename (the part after
|
|
// $RUNNER_TEMP/) of any $RUNNER_TEMP-relative path a run: block mentions,
|
|
// including one built from a shell loop variable (e.g. bazel-bep-$step.json).
|
|
var ciAnalyticsRunnerTempPathRE = regexp.MustCompile(`\$RUNNER_TEMP/([\w.$-]+)`)
|
|
|
|
// ciAnalyticsWriterFlagRE captures the $RUNNER_TEMP-relative basename
|
|
// written by one of the three bazel flags that produce a BEP file, a
|
|
// compact exec log or a JSON profile: the only three file kinds the
|
|
// summary step is allowed to read.
|
|
var ciAnalyticsWriterFlagRE = regexp.MustCompile(`--(?:build_event_json_file|execution_log_compact_file|profile)="?\$RUNNER_TEMP/([\w.$-]+)"?`)
|
|
|
|
// ciAnalyticsForLoopRE matches a `for <var> in <values>; do` line, so a
|
|
// basename built from the loop variable (bazel-bep-$step.json) can be
|
|
// expanded to every concrete value the loop actually iterates.
|
|
var ciAnalyticsForLoopRE = regexp.MustCompile(`for (\w+) in ([^;]+); do`)
|
|
|
|
// ciAnalyticsFileBasenameRE keeps only basenames that name a real
|
|
// input file (.json or .log.zst): the summary step's own --out path
|
|
// ($RUNNER_TEMP/ci-analytics/ci-analytics.json) is not $RUNNER_TEMP-direct
|
|
// and never matches a writer flag, so it is dropped rather than
|
|
// misclassified as a bad reader reference.
|
|
var ciAnalyticsFileBasenameRE = regexp.MustCompile(`\.(json|log\.zst)$`)
|
|
|
|
const (
|
|
// The path/retention/cleanup contract of every "Upload CI analytics
|
|
// summary" step: the redacted summary ci_analytics_extract.py wrote,
|
|
// kept a week, replaced (not 409'd) by a job re-run, and silently
|
|
// skipped (not failed) if the summary step itself never ran.
|
|
ciAnalyticsUploadPath = "${{ runner.temp }}/ci-analytics/ci-analytics.json"
|
|
ciAnalyticsUploadRetentionDays = "7"
|
|
ciAnalyticsUploadIfNoFilesFound = "ignore"
|
|
ciAnalyticsUploadOverwrite = "true"
|
|
ciAnalyticsUploadTimeoutMinutes = 2
|
|
ciAnalyticsSummaryMaxTimeout = 3
|
|
// The real job context (requires actionlint >= 1.7.8), not a
|
|
// github.run_id substitution: the design's contract with S4.
|
|
ciAnalyticsCheckRunID = "${{ job.check_run_id }}"
|
|
ciAnalyticsMode = "${{ needs.rbe.outputs.mode }}"
|
|
ciAnalyticsPRHint = "${{ github.event.pull_request.number || 0 }}"
|
|
)
|
|
|
|
func ciAnalyticsContinueOnError(v any) bool {
|
|
switch v := v.(type) {
|
|
case bool:
|
|
return v
|
|
case string:
|
|
return v == "true"
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// ciAnalyticsExpandBasenames expands every $<loopvar> token in basenames
|
|
// using any `for <loopvar> in ...; do` loop found in run, returning the
|
|
// set of concrete basenames actually produced or consumed. A basename with
|
|
// no loop-variable token (the 7 lanes with a single literal BEP/exec-log/
|
|
// profile set) expands to itself unchanged.
|
|
func ciAnalyticsExpandBasenames(run string, basenames []string) map[string]bool {
|
|
out := map[string]bool{}
|
|
loops := ciAnalyticsForLoopRE.FindAllStringSubmatch(run, -1)
|
|
for _, b := range basenames {
|
|
expanded := []string{b}
|
|
for _, loop := range loops {
|
|
token := "$" + loop[1]
|
|
if !strings.Contains(b, token) {
|
|
continue
|
|
}
|
|
var next []string
|
|
for _, e := range expanded {
|
|
for _, v := range strings.Fields(loop[2]) {
|
|
next = append(next, strings.ReplaceAll(e, token, v))
|
|
}
|
|
}
|
|
expanded = next
|
|
}
|
|
for _, e := range expanded {
|
|
out[e] = true
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// TestCIAnalyticsStepsAreSafeAndBounded is beads' version of gascity's pin
|
|
// test of the same name (scripts/bazel_multilane_test.go, ci/analytics-
|
|
// extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac), adapted to beads'
|
|
// 8 separately-named lane jobs rather than gascity's single matrix "lane"
|
|
// job plus "coverage". The rbe-ci-bep-analytics-design.md "CI analytics
|
|
// summary" step (ci_analytics_extract.py) and its upload, in every lane
|
|
// job, must stay strictly reporting-only: continue-on-error, a short
|
|
// timeout so a parser bug can never make the step (or the job) run long,
|
|
// no raw GitHub Actions expression spliced into the shell command (only
|
|
// validated $ENV_VARS, never a string this extractor would have to
|
|
// re-validate itself), !cancelled() required rather than always() (always()
|
|
// also runs after a workflow cancellation, which these reporting-only
|
|
// steps should not survive), an exact upload name/path/retention contract,
|
|
// and every path the summary step reads must actually be written by a
|
|
// bazel flag earlier in the same job.
|
|
func TestCIAnalyticsStepsAreSafeAndBounded(t *testing.T) {
|
|
workflow := readCIWorkflow(t, "bazel.yml")
|
|
|
|
// Every job in bazel.yml is either one of the lane jobs or has a
|
|
// recorded reason it is excluded: a new job cannot join bazel.yml
|
|
// without a decision about CI analytics.
|
|
for name := range workflow.Jobs {
|
|
_, lane := ciAnalyticsLaneJobs[name]
|
|
_, excluded := ciAnalyticsExcludedJobs[name]
|
|
if lane == excluded {
|
|
t.Errorf("bazel.yml job %q: lane=%v excluded=%v; add it to exactly one of ciAnalyticsLaneJobs or ciAnalyticsExcludedJobs (with a reason)", name, lane, excluded)
|
|
}
|
|
}
|
|
for name := range ciAnalyticsLaneJobs {
|
|
if _, ok := workflow.Jobs[name]; !ok {
|
|
t.Errorf("ciAnalyticsLaneJobs lists %s, which bazel.yml does not have", name)
|
|
}
|
|
}
|
|
for name := range ciAnalyticsExcludedJobs {
|
|
if _, ok := workflow.Jobs[name]; !ok {
|
|
t.Errorf("ciAnalyticsExcludedJobs lists %s, which bazel.yml does not have", name)
|
|
}
|
|
}
|
|
|
|
// --execution_log_compact_file without
|
|
// --experimental_build_event_upload_strategy=local in the same command
|
|
// writes a compact exec log bazel never uploads anywhere the extractor
|
|
// (or anything else) can read it back: require both together, in every
|
|
// job's every step, not just the 8 lane jobs.
|
|
for name, job := range workflow.Jobs {
|
|
for _, step := range job.Steps {
|
|
if strings.Contains(step.Run, "--execution_log_compact_file") && !strings.Contains(step.Run, "--experimental_build_event_upload_strategy=local") {
|
|
t.Errorf("%s job step %q: run has --execution_log_compact_file without --experimental_build_event_upload_strategy=local in the same command", name, step.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
for jobName, wantLane := range ciAnalyticsLaneJobs {
|
|
job := workflow.job(t, jobName)
|
|
|
|
summaryIdx := slices.IndexFunc(job.Steps, func(s ciWorkflowStep) bool { return s.Name == "CI analytics summary" })
|
|
uploadIdx := slices.IndexFunc(job.Steps, func(s ciWorkflowStep) bool { return s.Name == "Upload CI analytics summary" })
|
|
if summaryIdx < 0 || uploadIdx < 0 {
|
|
t.Fatalf("%s job: missing CI analytics summary/upload steps", jobName)
|
|
}
|
|
if uploadIdx != summaryIdx+1 {
|
|
t.Errorf("%s job: Upload CI analytics summary is step %d, want %d (immediately after CI analytics summary)", jobName, uploadIdx, summaryIdx+1)
|
|
}
|
|
summary := job.Steps[summaryIdx]
|
|
upload := job.Steps[uploadIdx]
|
|
|
|
if !ciAnalyticsContinueOnError(summary.ContinueOnError) {
|
|
t.Errorf("%s job %q: continue-on-error %v, want true", jobName, summary.Name, summary.ContinueOnError)
|
|
}
|
|
if summary.TimeoutMinutes <= 0 || summary.TimeoutMinutes > ciAnalyticsSummaryMaxTimeout {
|
|
t.Errorf("%s job %q: timeout-minutes %d, want 1-%d", jobName, summary.Name, summary.TimeoutMinutes, ciAnalyticsSummaryMaxTimeout)
|
|
}
|
|
if ciAnalyticsRunTemplateRE.MatchString(summary.Run) {
|
|
t.Errorf("%s job %q: run: contains a ${{ }} expression; use env: and a shell variable instead:\n%s", jobName, summary.Name, summary.Run)
|
|
}
|
|
//nolint:misspell // GitHub Actions spells it cancelled()
|
|
if !strings.Contains(summary.If, "!cancelled()") {
|
|
t.Errorf("%s job %q: if %q does not require !cancelled(); always() also runs after a workflow cancellation", jobName, summary.Name, summary.If)
|
|
}
|
|
if got := summary.Env["LANE"]; got != wantLane {
|
|
t.Errorf("%s job CI analytics summary: LANE %q, want %q", jobName, got, wantLane)
|
|
}
|
|
if got := summary.Env["MODE"]; got != ciAnalyticsMode {
|
|
t.Errorf("%s job CI analytics summary: MODE %q, want %q", jobName, got, ciAnalyticsMode)
|
|
}
|
|
if got := summary.Env["CHECK_RUN_ID"]; got != ciAnalyticsCheckRunID {
|
|
t.Errorf("%s job CI analytics summary: CHECK_RUN_ID %q, want %q", jobName, got, ciAnalyticsCheckRunID)
|
|
}
|
|
if got := summary.Env["PR_HINT"]; got != ciAnalyticsPRHint {
|
|
t.Errorf("%s job CI analytics summary: PR_HINT %q, want %q", jobName, got, ciAnalyticsPRHint)
|
|
}
|
|
|
|
if !ciAnalyticsContinueOnError(upload.ContinueOnError) {
|
|
t.Errorf("%s job %q: continue-on-error %v, want true", jobName, upload.Name, upload.ContinueOnError)
|
|
}
|
|
if upload.TimeoutMinutes != ciAnalyticsUploadTimeoutMinutes {
|
|
t.Errorf("%s job %q: timeout-minutes %d, want %d", jobName, upload.Name, upload.TimeoutMinutes, ciAnalyticsUploadTimeoutMinutes)
|
|
}
|
|
//nolint:misspell // GitHub Actions spells it cancelled()
|
|
if !strings.Contains(upload.If, "!cancelled()") {
|
|
t.Errorf("%s job %q: if %q does not require !cancelled()", jobName, upload.Name, upload.If)
|
|
}
|
|
wantUploadName := "ci-analytics-" + wantLane + "-${{ github.run_attempt }}"
|
|
if upload.With["name"] != wantUploadName {
|
|
t.Errorf("%s job %q: upload name %q, want %q", jobName, upload.Name, upload.With["name"], wantUploadName)
|
|
}
|
|
if upload.With["path"] != ciAnalyticsUploadPath {
|
|
t.Errorf("%s job %q: upload path %q, want %q (the redacted summary, never a raw BEP/exec-log/profile)", jobName, upload.Name, upload.With["path"], ciAnalyticsUploadPath)
|
|
}
|
|
if upload.With["retention-days"] != ciAnalyticsUploadRetentionDays {
|
|
t.Errorf("%s job %q: retention-days %q, want %q", jobName, upload.Name, upload.With["retention-days"], ciAnalyticsUploadRetentionDays)
|
|
}
|
|
if upload.With["if-no-files-found"] != ciAnalyticsUploadIfNoFilesFound {
|
|
t.Errorf("%s job %q: if-no-files-found %q, want %q", jobName, upload.Name, upload.With["if-no-files-found"], ciAnalyticsUploadIfNoFilesFound)
|
|
}
|
|
if upload.With["overwrite"] != ciAnalyticsUploadOverwrite {
|
|
t.Errorf("%s job %q: overwrite %q, want %q", jobName, upload.Name, upload.With["overwrite"], ciAnalyticsUploadOverwrite)
|
|
}
|
|
|
|
// Every $RUNNER_TEMP file the summary step reads must be written by
|
|
// a --build_event_json_file/--execution_log_compact_file/--profile
|
|
// flag somewhere earlier in the same job (expanding any shell loop
|
|
// variable on both sides, e.g. bazel-pure's per-step bazel-bep-
|
|
// $step.json).
|
|
readerRaw := ciAnalyticsRunnerTempPathRE.FindAllStringSubmatch(summary.Run, -1)
|
|
readerBasenames := make([]string, 0, len(readerRaw))
|
|
for _, m := range readerRaw {
|
|
readerBasenames = append(readerBasenames, m[1])
|
|
}
|
|
readerSet := ciAnalyticsExpandBasenames(summary.Run, readerBasenames)
|
|
|
|
writerSet := map[string]bool{}
|
|
for i, step := range job.Steps {
|
|
if i == summaryIdx || i == uploadIdx {
|
|
continue
|
|
}
|
|
raw := ciAnalyticsWriterFlagRE.FindAllStringSubmatch(step.Run, -1)
|
|
basenames := make([]string, 0, len(raw))
|
|
for _, m := range raw {
|
|
basenames = append(basenames, m[1])
|
|
}
|
|
for b := range ciAnalyticsExpandBasenames(step.Run, basenames) {
|
|
writerSet[b] = true
|
|
}
|
|
}
|
|
|
|
for b := range readerSet {
|
|
if !ciAnalyticsFileBasenameRE.MatchString(b) {
|
|
continue // not a file reference (e.g. the --out path's directory component)
|
|
}
|
|
if !writerSet[b] {
|
|
t.Errorf("%s job CI analytics summary: reads %q, which no --build_event_json_file/--execution_log_compact_file/--profile flag in this job writes", jobName, b)
|
|
}
|
|
}
|
|
}
|
|
}
|