From 8caa7899a893da89f67b21b1e0639b85a634d519 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 08:25:40 +0000 Subject: [PATCH] ci(bazel): add CI analytics extraction to all lanes (S3) tools/bazel/ci_analytics_extract.py turns a lane's BEP file, compact execution log and JSON profile into a redacted ci-analytics-summary (timing, per-mnemonic cache classes, cold actions, test results, repo-fetch time): only validated labels, mnemonics, enums and numbers leave the runner, never an endpoint, cert/key path, env value, token or hostname. Every lane's "CI analytics summary" step runs the extractor and uploads its output as ci-analytics-- (7-day retention); CHECK_RUN_ID uses the real job.check_run_id context (requires actionlint >= 1.7.8), and both the summary and upload steps are continue-on-error, the summary capped at timeout-minutes: 3, and both gated on !cancelled() (not always(), which also runs after a workflow cancellation). The extractor, its 62-case unittest file, and testdata/ci_analytics/ (the real/* Bazel 9.2.0 fixtures, poisoned-bep.json, mismatch.bep.jsonl and regen.sh) are vendored byte-identical from gascity's ci/analytics-extractor slice (final squashed commit 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac), which is canonical for this pair the same way tools/rbe/cache-zstd-probe.sh is a byte copy. Fixtures are regenerated under env -i with every BEP field stripped unless it's on an explicit allowlist, and the compact exec log is committed decompressed (*.binpb) rather than as a zstd frame, so nothing opaque reaches version control. scripts/ci_analytics_extract_test.go is the thin Go wrapper: TestCIAnalyticsExtractorIsPinnedToGascity pins both vendored files' sha256 so local drift is caught, and TestCIAnalyticsExtractUnitTests runs gascity's unittest file in place. scripts/ci_analytics_workflow_test.go's TestCIAnalyticsStepsAreSafeAndBounded mirrors gascity's pin test, adapted to beads' 8 separately-named lane jobs: continue-on-error, timeout-minutes bounds, no raw ${{ }} expression in run:, !cancelled() required in if:, correct per-lane upload artifact name/path, and CHECK_RUN_ID == ${{ job.check_run_id }}, for every lane. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/bazel.yml | 315 +++- scripts/BUILD.bazel | 2 + scripts/ci_analytics_extract_test.go | 150 ++ scripts/ci_analytics_workflow_test.go | 282 ++++ scripts/ci_merge_queue_test.go | 13 + scripts/ci_workflow_test.go | 62 +- scripts/pr_lanes_bazel_coverage_test.go | 24 +- tools/bazel/ci_analytics_extract.py | 1276 +++++++++++++++ tools/bazel/ci_analytics_extract_test.py | 1429 +++++++++++++++++ .../testdata/ci_analytics/mismatch.bep.jsonl | 3 + .../testdata/ci_analytics/poisoned-bep.json | 8 + .../testdata/ci_analytics/real/cold-bep.json | 12 + .../ci_analytics/real/cold-exec.binpb | 549 +++++++ .../ci_analytics/real/cold-profile.json | 170 ++ .../ci_analytics/real/postclean-bep.json | 12 + .../ci_analytics/real/postclean-exec.binpb | 567 +++++++ tools/bazel/testdata/ci_analytics/regen.sh | 325 ++++ 17 files changed, 5186 insertions(+), 13 deletions(-) create mode 100644 scripts/ci_analytics_extract_test.go create mode 100644 scripts/ci_analytics_workflow_test.go create mode 100644 tools/bazel/ci_analytics_extract.py create mode 100644 tools/bazel/ci_analytics_extract_test.py create mode 100644 tools/bazel/testdata/ci_analytics/mismatch.bep.jsonl create mode 100644 tools/bazel/testdata/ci_analytics/poisoned-bep.json create mode 100644 tools/bazel/testdata/ci_analytics/real/cold-bep.json create mode 100644 tools/bazel/testdata/ci_analytics/real/cold-exec.binpb create mode 100644 tools/bazel/testdata/ci_analytics/real/cold-profile.json create mode 100644 tools/bazel/testdata/ci_analytics/real/postclean-bep.json create mode 100644 tools/bazel/testdata/ci_analytics/real/postclean-exec.binpb create mode 100755 tools/bazel/testdata/ci_analytics/regen.sh diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 4b74bd4a0..44f3de19a 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -785,6 +785,8 @@ jobs: bazel test //... --config=ci ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --profile="$RUNNER_TEMP/bazel-profile.json" \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" echo >> "$GITHUB_STEP_SUMMARY" @@ -919,6 +921,42 @@ jobs: path: ${{ runner.temp }}/bazel-ci-cache key: bazel-repo-v3-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + # A redacted, schema'd summary of this invocation's BEP, exec log and + # profile: timing, per-mnemonic cache classes, cold actions, test + # results, repo-fetch time. tools/bazel/ci_analytics_extract.py reads + # only a field whitelist and validates every emitted string, so this + # (unlike the raw BEP file) is safe to upload unredacted; see its + # module docstring. Mirrors gascity's ci/analytics-extractor lane step. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: test + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-test-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -994,9 +1032,14 @@ jobs: RBE_FORK_PR: ${{ github.event.pull_request.number }} - name: Run js/wasm hook boundary + id: build-wasm run: | set -euo pipefail - bazel build --config=js-wasm //internal/hooks:hooks_test + bazel build --config=js-wasm //internal/hooks:hooks_test \ + --profile="$RUNNER_TEMP/bazel-profile-build-wasm.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep-build-wasm.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-build-wasm.log.zst" \ + --experimental_build_event_upload_strategy=local bin="$(bazel cquery --config=js-wasm --output=files //internal/hooks:hooks_test)" cd internal/hooks @@ -1036,12 +1079,17 @@ jobs: fi - name: Build cmd/bd and pure-Go test binaries (--config=pure) + id: build-pure run: >- bazel build --config=pure //cmd/bd:bd //cmd/bd:bd_test //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test + --profile="$RUNNER_TEMP/bazel-profile-build-pure.json" + --build_event_json_file="$RUNNER_TEMP/bazel-bep-build-pure.json" + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-build-pure.log.zst" + --experimental_build_event_upload_strategy=local # Every release target (scripts/ci/release-targets.txt) on every PR, so # a platform-gating mistake is caught here instead of at tag time @@ -1063,28 +1111,81 @@ jobs: # starts in the next step. -test.run=^$ runs no test (TestMain still # runs; BEADS_TEST_SKIP=dolt keeps tracker's from starting a container). - name: Start every pure-Go artifact (gozstd contamination check) + id: test-pure run: | set -euo pipefail + # Not instrumented: no BEP/exec-log/profile is captured for this + # one-off smoke invocation (design's bazel-pure invocation list + # names build-wasm, build-pure, test-pure and test-bd only). bazel run --config=pure //cmd/bd:bd -- version bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ //internal/storage/embeddeddolt:embeddeddolt_test \ //internal/tracker:tracker_test \ --test_sharding_strategy=disabled \ '--test_arg=-test.run=^$' \ - --test_env=BEADS_TEST_SKIP=dolt + --test_env=BEADS_TEST_SKIP=dolt \ + --profile="$RUNNER_TEMP/bazel-profile-test-pure.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep-test-pure.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-test-pure.log.zst" \ + --experimental_build_event_upload_strategy=local # One unsharded process, like `go test -run`; the test.xml check keeps a # selector that matches nothing from passing. - name: Run pure-Go cmd/bd test subset (--config=pure) + id: test-bd run: | set -euo pipefail bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} //cmd/bd:bd_test \ --test_sharding_strategy=disabled \ - "--test_arg=-test.run=$PURE_CMD_BD_TESTS" + "--test_arg=-test.run=$PURE_CMD_BD_TESTS" \ + --profile="$RUNNER_TEMP/bazel-profile-test-bd.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep-test-bd.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-test-bd.log.zst" \ + --experimental_build_event_upload_strategy=local n="$(grep -c ' 0 )) + # A redacted, schema'd summary of this job's four BEP-tracked + # invocations (build-wasm, build-pure, test-pure, test-bd; `bazel run + # ... -- version` above is deliberately not tracked, see its comment). + # Same contract as bazel-test's step; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled()" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: pure + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + args=() + for step in build-wasm build-pure test-pure test-bd; do + bep="$RUNNER_TEMP/bazel-bep-$step.json" + [ -f "$bep" ] || continue + args+=(--invocation "$step" "$bep" "$RUNNER_TEMP/bazel-exec-$step.log.zst" "$RUNNER_TEMP/bazel-profile-$step.json") + done + [ "${#args[@]}" -gt 0 ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + "${args[@]}" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-pure-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1198,6 +1299,9 @@ jobs: rc=0 bazel test //... --config=embedded ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1239,6 +1343,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: embedded + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-embedded-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1341,6 +1476,9 @@ jobs: fi bazel test //... --config=integration ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" @@ -1366,6 +1504,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: integration + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-integration-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1431,7 +1600,12 @@ jobs: set -o pipefail start=$(date +%s) rc=0 - bazel test //... --config=doltserver ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + bazel test //... --config=doltserver ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1447,6 +1621,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: doltserver + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-doltserver-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1562,6 +1767,9 @@ jobs: rc=0 bazel test //... --config=doltserver-proxied ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver-proxied: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1593,6 +1801,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: proxied-server + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-proxied-server-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1673,6 +1912,9 @@ jobs: rc=0 bazel test //... --config=doltserver-integration ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver-integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1715,6 +1957,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: server-storage + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-server-storage-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1797,6 +2070,9 @@ jobs: rc=0 bazel test //... --config=doltserver-cmd ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver-cmd: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1820,6 +2096,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: cmd-dolt + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-cmd-dolt-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too; job.status is the steps' status, unaffected by the diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index cfaad6e93..c6c4caa81 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -28,6 +28,8 @@ go_test( "check_testing_short_test.go", "check_winget_portable_alias_test.go", "ci_actions_policy_test.go", + "ci_analytics_extract_test.go", + "ci_analytics_workflow_test.go", "ci_blacksmith_runner_test.go", "ci_capability_selector_test.go", "ci_f7b_review_test.go", diff --git a/scripts/ci_analytics_extract_test.go b/scripts/ci_analytics_extract_test.go new file mode 100644 index 000000000..db5fc463b --- /dev/null +++ b/scripts/ci_analytics_extract_test.go @@ -0,0 +1,150 @@ +package scripts_test + +import ( + "crypto/sha256" + "encoding/hex" + "io/fs" + "os" + "os/exec" + "path/filepath" + "sort" + "testing" +) + +// ciAnalyticsExtractScript and ciAnalyticsExtractUnitTest are a byte copy of +// gascity's tools/bazel/ci_analytics_extract.py and +// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extractor, +// gascity commit 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac, the final +// squashed-to-one-commit history superseding 32d51ce26a and the original +// 9104311a8a after the S2 review found the first cut's real/*-bep.json +// fixtures leaked operator env: a token, SSH IPs, the OpenBao address, the +// hostname and home paths. The fixtures are generated under env -i and +// committed with the compact exec log decompressed (*.binpb), never a +// zstd frame, so nothing opaque reaches version control), the same way +// tools/rbe/cache-zstd-probe.sh is a byte copy. gascity is canonical for +// this pair; TestCIAnalyticsExtractorIsPinnedToGascity below pins both +// files' sha256 so a local edit here is caught instead of silently +// drifting out of lockstep. The vendored test file's own 62 cases (run in +// place by TestCIAnalyticsExtractUnitTests) now include the +// mismatched-invocation-id and truncated-BEP cases beads originally added +// (test_beads_mismatch_fixture_reports_mismatch, +// test_truncated_last_line_sets_bep_truncated), so no separate Go-side +// duplicate of either remains here. +func ciAnalyticsExtractScript(root string) string { + return filepath.Join(root, "tools", "bazel", "ci_analytics_extract.py") +} + +func ciAnalyticsExtractUnitTest(root string) string { + return filepath.Join(root, "tools", "bazel", "ci_analytics_extract_test.py") +} + +func ciAnalyticsTestdataDir(root string) string { + return filepath.Join(root, "tools", "bazel", "testdata", "ci_analytics") +} + +// ciAnalyticsExtractSHA256 and ciAnalyticsExtractUnitTestSHA256 pin the +// vendored files' content, computed from gascity's final sanitized, +// squashed-to-one-commit history at 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac. +const ( + ciAnalyticsExtractSHA256 = "6bb912b17faac59a12ec53f71fbfbedfecc503037b6f5954e878d108abaae44f" + ciAnalyticsExtractUnitTestSHA256 = "e833e1c961010cd02189afaef273a261a54ccabe916b03bae3115177378a405b" + // ciAnalyticsTestdataSHA256 pins testdata/ci_analytics/** as a whole + // (regen.sh, mismatch.bep.jsonl, poisoned-bep.json and real/*): a sha256 + // over every file's slash-joined relative path and bytes, each + // NUL-terminated, sorted by path (ciAnalyticsTreeSHA256 below) - + // computed the same way over gascity's 00161373bc tree and confirmed + // identical, so this one hash catches a dropped, renamed or + // individually-edited fixture that a single-file sha256 (the two + // above) can't. + ciAnalyticsTestdataSHA256 = "dc9ac595df095ef8fe1b965d0e2a8d0b0e5694c3122208891d27655bcc2285d5" +) + +// ciAnalyticsTreeSHA256 hashes every regular file under dir: each entry +// contributes its dir-relative, slash-joined path, a NUL, its bytes and a +// NUL, in path-sorted order, so the result depends on the exact set of +// files present (not just their union of bytes) and is independent of the +// OS's directory-walk order. +func ciAnalyticsTreeSHA256(t *testing.T, dir string) string { + t.Helper() + var paths []string + if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + paths = append(paths, filepath.ToSlash(rel)) + return nil + }); err != nil { + t.Fatalf("walking %s: %v", dir, err) + } + sort.Strings(paths) + + h := sha256.New() + for _, rel := range paths { + raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("reading %s: %v", rel, err) + } + h.Write([]byte(rel)) + h.Write([]byte{0}) + h.Write(raw) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// TestCIAnalyticsExtractorIsPinnedToGascity fails the moment either vendored +// file is edited locally without also updating the pin (and, ideally, +// upstreaming the change to gascity first): the design treats gascity as +// canonical for this pair, like cache-zstd-probe.sh. +func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) { + root := sourceRepoRoot(t) + for _, tc := range []struct { + path string + want string + }{ + {ciAnalyticsExtractScript(root), ciAnalyticsExtractSHA256}, + {ciAnalyticsExtractUnitTest(root), ciAnalyticsExtractUnitTestSHA256}, + } { + raw, err := os.ReadFile(tc.path) + if err != nil { + t.Fatalf("reading %s: %v", tc.path, err) + } + sum := sha256.Sum256(raw) + if got := hex.EncodeToString(sum[:]); got != tc.want { + t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac): "+ + "either this file drifted from gascity's canonical copy, or gascity changed it "+ + "and this pin (and the vendored copy) needs updating to match", tc.path, got, tc.want) + } + } + + testdataDir := ciAnalyticsTestdataDir(root) + if got := ciAnalyticsTreeSHA256(t, testdataDir); got != ciAnalyticsTestdataSHA256 { + t.Errorf("%s tree sha256 = %s, want %s (gascity ci/analytics-extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac): "+ + "a fixture under testdata/ci_analytics/ was added, removed, renamed or edited without updating this pin", testdataDir, got, ciAnalyticsTestdataSHA256) + } +} + +// TestCIAnalyticsExtractUnitTests runs gascity's own 62 unittest cases +// in-place against the vendored extractor: the real Bazel 9.2.0 fixtures +// (testdata/ci_analytics/real/*, regenerated under env -i after the S2 +// leak), the synthetic remote/cache-hit exec log built by the test's own +// varint encoder, the poisoned/malformed BEP and exec-log robustness cases, +// and the ported mismatched-invocation-id and truncated-BEP cases. This is +// the "thin Go wrapper" around the vendored pair. +func TestCIAnalyticsExtractUnitTests(t *testing.T) { + root := sourceRepoRoot(t) + python := requireHostTool(t, "python3") + cmd := exec.Command(python, ciAnalyticsExtractUnitTest(root), "-v") + cmd.Dir = root + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("ci_analytics_extract_test.py failed: %v\n%s", err, out) + } +} diff --git a/scripts/ci_analytics_workflow_test.go b/scripts/ci_analytics_workflow_test.go new file mode 100644 index 000000000..d94a94c16 --- /dev/null +++ b/scripts/ci_analytics_workflow_test.go @@ -0,0 +1,282 @@ +package scripts_test + +import ( + "regexp" + "slices" + "strings" + "testing" +) + +// ciAnalyticsLaneJobs lists bazel.yml's lane jobs that carry the "CI +// analytics summary"/"Upload CI analytics summary" steps +// (rbe-ci-bep-analytics-design.md, S3), mapped to each job's LANE env +// value. The artifact name and the LANE env are both derived from this one +// map in the checks below, instead of being pinned independently and +// allowed to drift apart. +var ciAnalyticsLaneJobs = map[string]string{ + "bazel-test": "test", + "bazel-pure": "pure", + "bazel-embedded": "embedded", + "bazel-integration": "integration", + "bazel-doltserver": "doltserver", + "bazel-proxied": "proxied-server", + "bazel-server-storage": "server-storage", + "bazel-cmd-dolt": "cmd-dolt", +} + +// ciAnalyticsExcludedJobs are bazel.yml jobs that deliberately do not run +// the CI analytics extractor, with the reason: mirrors +// TestBazelLaneIsGatedAlongsideLegacy's gated/advisory/flag-gated split, so +// a new job in bazel.yml can't silently go unaccounted for here either. +var ciAnalyticsExcludedJobs = map[string]string{ + "rbe": "decides the execution mode; builds nothing itself, so there is no BEP/exec-log/profile to summarize", + "rbe-prewarm": "best-effort pre-warm dispatch to gascity's rbe-west pool; not a build/test lane", + "package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract", + "package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract", +} + +// ciAnalyticsRunTemplateRE finds a GitHub Actions expression (${{ ... }}) +// inside a `run:` block: ci_analytics_extract.py's arguments must come from +// $LANE/$MODE/$CHECK_RUN_ID/$PR_HINT shell variables (set in `env:`, which +// GitHub Actions substitutes before the shell ever sees them), never a raw +// ${{ }} expression spliced into the command line itself. +var ciAnalyticsRunTemplateRE = regexp.MustCompile(`\$\{\{`) + +// ciAnalyticsRunnerTempPathRE captures the basename (the part after +// $RUNNER_TEMP/) of any $RUNNER_TEMP-relative path a run: block mentions, +// including one built from a shell loop variable (e.g. bazel-bep-$step.json). +var ciAnalyticsRunnerTempPathRE = regexp.MustCompile(`\$RUNNER_TEMP/([\w.$-]+)`) + +// ciAnalyticsWriterFlagRE captures the $RUNNER_TEMP-relative basename +// written by one of the three bazel flags that produce a BEP file, a +// compact exec log or a JSON profile: the only three file kinds the +// summary step is allowed to read. +var ciAnalyticsWriterFlagRE = regexp.MustCompile(`--(?:build_event_json_file|execution_log_compact_file|profile)="?\$RUNNER_TEMP/([\w.$-]+)"?`) + +// ciAnalyticsForLoopRE matches a `for in ; do` line, so a +// basename built from the loop variable (bazel-bep-$step.json) can be +// expanded to every concrete value the loop actually iterates. +var ciAnalyticsForLoopRE = regexp.MustCompile(`for (\w+) in ([^;]+); do`) + +// ciAnalyticsFileBasenameRE keeps only basenames that name a real +// input file (.json or .log.zst): the summary step's own --out path +// ($RUNNER_TEMP/ci-analytics/ci-analytics.json) is not $RUNNER_TEMP-direct +// and never matches a writer flag, so it is dropped rather than +// misclassified as a bad reader reference. +var ciAnalyticsFileBasenameRE = regexp.MustCompile(`\.(json|log\.zst)$`) + +const ( + // The path/retention/cleanup contract of every "Upload CI analytics + // summary" step: the redacted summary ci_analytics_extract.py wrote, + // kept a week, replaced (not 409'd) by a job re-run, and silently + // skipped (not failed) if the summary step itself never ran. + ciAnalyticsUploadPath = "${{ runner.temp }}/ci-analytics/ci-analytics.json" + ciAnalyticsUploadRetentionDays = "7" + ciAnalyticsUploadIfNoFilesFound = "ignore" + ciAnalyticsUploadOverwrite = "true" + ciAnalyticsUploadTimeoutMinutes = 2 + ciAnalyticsSummaryMaxTimeout = 3 + // The real job context (requires actionlint >= 1.7.8), not a + // github.run_id substitution: the design's contract with S4. + ciAnalyticsCheckRunID = "${{ job.check_run_id }}" + ciAnalyticsMode = "${{ needs.rbe.outputs.mode }}" + ciAnalyticsPRHint = "${{ github.event.pull_request.number || 0 }}" +) + +func ciAnalyticsContinueOnError(v any) bool { + switch v := v.(type) { + case bool: + return v + case string: + return v == "true" + default: + return false + } +} + +// ciAnalyticsExpandBasenames expands every $ token in basenames +// using any `for in ...; do` loop found in run, returning the +// set of concrete basenames actually produced or consumed. A basename with +// no loop-variable token (the 7 lanes with a single literal BEP/exec-log/ +// profile set) expands to itself unchanged. +func ciAnalyticsExpandBasenames(run string, basenames []string) map[string]bool { + out := map[string]bool{} + loops := ciAnalyticsForLoopRE.FindAllStringSubmatch(run, -1) + for _, b := range basenames { + expanded := []string{b} + for _, loop := range loops { + token := "$" + loop[1] + if !strings.Contains(b, token) { + continue + } + var next []string + for _, e := range expanded { + for _, v := range strings.Fields(loop[2]) { + next = append(next, strings.ReplaceAll(e, token, v)) + } + } + expanded = next + } + for _, e := range expanded { + out[e] = true + } + } + return out +} + +// TestCIAnalyticsStepsAreSafeAndBounded is beads' version of gascity's pin +// test of the same name (scripts/bazel_multilane_test.go, ci/analytics- +// extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac), adapted to beads' +// 8 separately-named lane jobs rather than gascity's single matrix "lane" +// job plus "coverage". The rbe-ci-bep-analytics-design.md "CI analytics +// summary" step (ci_analytics_extract.py) and its upload, in every lane +// job, must stay strictly reporting-only: continue-on-error, a short +// timeout so a parser bug can never make the step (or the job) run long, +// no raw GitHub Actions expression spliced into the shell command (only +// validated $ENV_VARS, never a string this extractor would have to +// re-validate itself), !cancelled() required rather than always() (always() +// also runs after a workflow cancellation, which these reporting-only +// steps should not survive), an exact upload name/path/retention contract, +// and every path the summary step reads must actually be written by a +// bazel flag earlier in the same job. +func TestCIAnalyticsStepsAreSafeAndBounded(t *testing.T) { + workflow := readCIWorkflow(t, "bazel.yml") + + // Every job in bazel.yml is either one of the lane jobs or has a + // recorded reason it is excluded: a new job cannot join bazel.yml + // without a decision about CI analytics. + for name := range workflow.Jobs { + _, lane := ciAnalyticsLaneJobs[name] + _, excluded := ciAnalyticsExcludedJobs[name] + if lane == excluded { + t.Errorf("bazel.yml job %q: lane=%v excluded=%v; add it to exactly one of ciAnalyticsLaneJobs or ciAnalyticsExcludedJobs (with a reason)", name, lane, excluded) + } + } + for name := range ciAnalyticsLaneJobs { + if _, ok := workflow.Jobs[name]; !ok { + t.Errorf("ciAnalyticsLaneJobs lists %s, which bazel.yml does not have", name) + } + } + for name := range ciAnalyticsExcludedJobs { + if _, ok := workflow.Jobs[name]; !ok { + t.Errorf("ciAnalyticsExcludedJobs lists %s, which bazel.yml does not have", name) + } + } + + // --execution_log_compact_file without + // --experimental_build_event_upload_strategy=local in the same command + // writes a compact exec log bazel never uploads anywhere the extractor + // (or anything else) can read it back: require both together, in every + // job's every step, not just the 8 lane jobs. + for name, job := range workflow.Jobs { + for _, step := range job.Steps { + if strings.Contains(step.Run, "--execution_log_compact_file") && !strings.Contains(step.Run, "--experimental_build_event_upload_strategy=local") { + t.Errorf("%s job step %q: run has --execution_log_compact_file without --experimental_build_event_upload_strategy=local in the same command", name, step.Name) + } + } + } + + for jobName, wantLane := range ciAnalyticsLaneJobs { + job := workflow.job(t, jobName) + + summaryIdx := slices.IndexFunc(job.Steps, func(s ciWorkflowStep) bool { return s.Name == "CI analytics summary" }) + uploadIdx := slices.IndexFunc(job.Steps, func(s ciWorkflowStep) bool { return s.Name == "Upload CI analytics summary" }) + if summaryIdx < 0 || uploadIdx < 0 { + t.Fatalf("%s job: missing CI analytics summary/upload steps", jobName) + } + if uploadIdx != summaryIdx+1 { + t.Errorf("%s job: Upload CI analytics summary is step %d, want %d (immediately after CI analytics summary)", jobName, uploadIdx, summaryIdx+1) + } + summary := job.Steps[summaryIdx] + upload := job.Steps[uploadIdx] + + if !ciAnalyticsContinueOnError(summary.ContinueOnError) { + t.Errorf("%s job %q: continue-on-error %v, want true", jobName, summary.Name, summary.ContinueOnError) + } + if summary.TimeoutMinutes <= 0 || summary.TimeoutMinutes > ciAnalyticsSummaryMaxTimeout { + t.Errorf("%s job %q: timeout-minutes %d, want 1-%d", jobName, summary.Name, summary.TimeoutMinutes, ciAnalyticsSummaryMaxTimeout) + } + if ciAnalyticsRunTemplateRE.MatchString(summary.Run) { + t.Errorf("%s job %q: run: contains a ${{ }} expression; use env: and a shell variable instead:\n%s", jobName, summary.Name, summary.Run) + } + //nolint:misspell // GitHub Actions spells it cancelled() + if !strings.Contains(summary.If, "!cancelled()") { + t.Errorf("%s job %q: if %q does not require !cancelled(); always() also runs after a workflow cancellation", jobName, summary.Name, summary.If) + } + if got := summary.Env["LANE"]; got != wantLane { + t.Errorf("%s job CI analytics summary: LANE %q, want %q", jobName, got, wantLane) + } + if got := summary.Env["MODE"]; got != ciAnalyticsMode { + t.Errorf("%s job CI analytics summary: MODE %q, want %q", jobName, got, ciAnalyticsMode) + } + if got := summary.Env["CHECK_RUN_ID"]; got != ciAnalyticsCheckRunID { + t.Errorf("%s job CI analytics summary: CHECK_RUN_ID %q, want %q", jobName, got, ciAnalyticsCheckRunID) + } + if got := summary.Env["PR_HINT"]; got != ciAnalyticsPRHint { + t.Errorf("%s job CI analytics summary: PR_HINT %q, want %q", jobName, got, ciAnalyticsPRHint) + } + + if !ciAnalyticsContinueOnError(upload.ContinueOnError) { + t.Errorf("%s job %q: continue-on-error %v, want true", jobName, upload.Name, upload.ContinueOnError) + } + if upload.TimeoutMinutes != ciAnalyticsUploadTimeoutMinutes { + t.Errorf("%s job %q: timeout-minutes %d, want %d", jobName, upload.Name, upload.TimeoutMinutes, ciAnalyticsUploadTimeoutMinutes) + } + //nolint:misspell // GitHub Actions spells it cancelled() + if !strings.Contains(upload.If, "!cancelled()") { + t.Errorf("%s job %q: if %q does not require !cancelled()", jobName, upload.Name, upload.If) + } + wantUploadName := "ci-analytics-" + wantLane + "-${{ github.run_attempt }}" + if upload.With["name"] != wantUploadName { + t.Errorf("%s job %q: upload name %q, want %q", jobName, upload.Name, upload.With["name"], wantUploadName) + } + if upload.With["path"] != ciAnalyticsUploadPath { + t.Errorf("%s job %q: upload path %q, want %q (the redacted summary, never a raw BEP/exec-log/profile)", jobName, upload.Name, upload.With["path"], ciAnalyticsUploadPath) + } + if upload.With["retention-days"] != ciAnalyticsUploadRetentionDays { + t.Errorf("%s job %q: retention-days %q, want %q", jobName, upload.Name, upload.With["retention-days"], ciAnalyticsUploadRetentionDays) + } + if upload.With["if-no-files-found"] != ciAnalyticsUploadIfNoFilesFound { + t.Errorf("%s job %q: if-no-files-found %q, want %q", jobName, upload.Name, upload.With["if-no-files-found"], ciAnalyticsUploadIfNoFilesFound) + } + if upload.With["overwrite"] != ciAnalyticsUploadOverwrite { + t.Errorf("%s job %q: overwrite %q, want %q", jobName, upload.Name, upload.With["overwrite"], ciAnalyticsUploadOverwrite) + } + + // Every $RUNNER_TEMP file the summary step reads must be written by + // a --build_event_json_file/--execution_log_compact_file/--profile + // flag somewhere earlier in the same job (expanding any shell loop + // variable on both sides, e.g. bazel-pure's per-step bazel-bep- + // $step.json). + readerRaw := ciAnalyticsRunnerTempPathRE.FindAllStringSubmatch(summary.Run, -1) + readerBasenames := make([]string, 0, len(readerRaw)) + for _, m := range readerRaw { + readerBasenames = append(readerBasenames, m[1]) + } + readerSet := ciAnalyticsExpandBasenames(summary.Run, readerBasenames) + + writerSet := map[string]bool{} + for i, step := range job.Steps { + if i == summaryIdx || i == uploadIdx { + continue + } + raw := ciAnalyticsWriterFlagRE.FindAllStringSubmatch(step.Run, -1) + basenames := make([]string, 0, len(raw)) + for _, m := range raw { + basenames = append(basenames, m[1]) + } + for b := range ciAnalyticsExpandBasenames(step.Run, basenames) { + writerSet[b] = true + } + } + + for b := range readerSet { + if !ciAnalyticsFileBasenameRE.MatchString(b) { + continue // not a file reference (e.g. the --out path's directory component) + } + if !writerSet[b] { + t.Errorf("%s job CI analytics summary: reads %q, which no --build_event_json_file/--execution_log_compact_file/--profile flag in this job writes", jobName, b) + } + } + } +} diff --git a/scripts/ci_merge_queue_test.go b/scripts/ci_merge_queue_test.go index 47de8d20f..354db638c 100644 --- a/scripts/ci_merge_queue_test.go +++ b/scripts/ci_merge_queue_test.go @@ -199,6 +199,19 @@ func TestMergeQueuePullRequestFieldsHandleMergeGroup(t *testing.T) { "detect-ci-tier.steps[2].env.PR_HEAD_SHA": "advisory shadow selector; merge_group selects everything", "bazel-test.steps[*].env.PR_NUMBER": "bazel-sync patch metadata; bazel-autofix.yml ignores non-pull_request runs", "bazel-test.steps[*].env.PR_HEAD_SHA": "bazel-sync patch metadata; bazel-autofix.yml ignores non-pull_request runs", + // bazel.yml's CI analytics summary step (rbe-ci-bep-analytics- + // design.md, S3), one entry per lane job (ciAnalyticsLaneJobs in + // ci_analytics_workflow_test.go) rather than a bare "steps[*]...", + // so this stays scoped to bazel.yml's own PR_HINT and can't + // silently also cover a future PR_HINT read elsewhere. + "bazel-test.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-pure.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-embedded.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-integration.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-doltserver.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-proxied.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-server-storage.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-cmd-dolt.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", } used := map[string]bool{} var fallbacks []string diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index afacfa1ba..ede9ca629 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2106,6 +2106,45 @@ const ( bazelRBEPrewarmContinueOnErrorPath = ".jobs." + bazelRBEPrewarmJobName + ".continue-on-error" ) +// rbe-ci-bep-analytics-design.md (S3) adds two steps to every lane job +// ("CI analytics summary" and "Upload CI analytics summary"): reporting +// only (the extracted summary feeds a follow-on analysis pipeline, nothing +// in bazel.yml or pr.yml reads its output), so both carry a deliberate +// step-level continue-on-error - the only step-level exception +// TestBazelWorkflowSecretsAndFailureSurface allows, matched below by job, +// step name and content (ciAnalyticsContinueOnErrorPaths) rather than by a +// single fixed path the way rbe-prewarm's job-level one is, since every +// lane job has its own copy of these two steps at a different index. + +// ciAnalyticsContinueOnErrorPaths returns the walkYAML path of every CI +// analytics step's continue-on-error key, so +// TestBazelWorkflowSecretsAndFailureSurface can allow exactly those and +// nothing else. Scoped three ways, not just by step name: the job must be +// one of ciAnalyticsLaneJobs (ci_analytics_workflow_test.go), the summary +// step's run must actually invoke ci_analytics_extract.py, and the upload +// step's artifact name must start with "ci-analytics-" - so a future step +// that happens to share one of these two names, in a job this design never +// touched, can't ride along on the exception unnoticed. +func ciAnalyticsContinueOnErrorPaths(t *testing.T) map[string]bool { + t.Helper() + paths := map[string]bool{} + for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if _, ok := ciAnalyticsLaneJobs[name]; !ok { + continue + } + for i, step := range job.Steps { + switch { + case step.Name == "CI analytics summary" && strings.Contains(step.Run, "ci_analytics_extract.py"): + case step.Name == "Upload CI analytics summary" && strings.HasPrefix(step.With["name"], "ci-analytics-"): + default: + continue + } + paths[fmt.Sprintf(".jobs.%s.steps[%d].continue-on-error", name, i)] = true + } + } + return paths +} + // rbe-fork: the lanes' setup-bazel env that asks it for a certificate (modes // fork-ro and fork-rw only), and the mint's status URL the rbe job probes. const ( @@ -4300,7 +4339,7 @@ func TestBazelIntegrationJob(t *testing.T) { } cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(test.Run, " ") // EXCLUDE_TARGETS: TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers. - const wantCmd = `bazel test //... --config=integration ` + bazelFreshArg + ` --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` + const wantCmd = `bazel test //... --config=integration ` + bazelFreshArg + ` --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" --profile="$RUNNER_TEMP/bazel-profile.json" --experimental_build_event_upload_strategy=local ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` if !strings.Contains(cmd, wantCmd) || !strings.Contains(test.Run, "set -o pipefail") { t.Errorf("%s test step does not run exactly %q:\n%s", bazelIntegJobName, wantCmd, test.Run) } @@ -4569,11 +4608,14 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { job, config, logs string wantSteps int }{ - {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 8}, + // +2 over the earlier counts: the CI analytics summary/upload steps + // (S3; see bazel-test's "CI analytics summary" comment), added to + // every lane job ahead of the result recorder. + {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 10}, // One more than bazel-proxied: a "Shard balance" step (rbe-ci-cost- // latency-study.md recommendation 4), since this is the tier whose // last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. - {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 9}, + {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 11}, } { job := workflow.job(t, c.job) if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 { @@ -4581,7 +4623,7 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { } assertBazelTierStep(t, job, c.job, c.config) if n := len(job.Steps); n != c.wantSteps { - t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, result recorder)", c.job, n, c.wantSteps) + t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps) } logs := job.step(t, "Upload test logs") if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { @@ -4621,6 +4663,9 @@ start=$(date +%s) rc=0 bazel test //... --config=embedded ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc"` @@ -5092,6 +5137,7 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { t.Errorf("%s job %s has %d setup-bazel steps, want 1", bazelWorkflowName, name, n) } } + ciAnalyticsPaths := ciAnalyticsContinueOnErrorPaths(t) secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { if key && value == "continue-on-error" { @@ -5101,9 +5147,13 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { // job's failure would still fail bazel.yml's own run and // cascade into pr.yml's ci-gate. So is each flag-gated lane's // (bazelFlagGatedLanes): advisory until its pr.yml flag is - // "true", and required then through its own gate id. + // "true", and required then through its own gate id. So are the + // two CI analytics steps in every lane job + // (bazelCIAnalyticsStepNames): reporting only, never a `needs` + // of anything and never read back by this workflow or its + // callers. _, flagGated := bazelFlagGatedLanes[strings.TrimSuffix(strings.TrimPrefix(path, ".jobs."), ".continue-on-error")] - if path != bazelRBEPrewarmContinueOnErrorPath && !flagGated { + if path != bazelRBEPrewarmContinueOnErrorPath && !flagGated && !ciAnalyticsPaths[path] { t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path) } } diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index d6b5ce0d8..5212bccf8 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -107,6 +107,8 @@ rc=0 bazel test //... --config=ci ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --profile="$RUNNER_TEMP/bazel-profile.json" \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" echo >> "$GITHUB_STEP_SUMMARY" @@ -114,17 +116,28 @@ exit "$rc"`, }, bazelPureJobName: { "Start every pure-Go artifact (gozstd contamination check)": `set -euo pipefail +# Not instrumented: no BEP/exec-log/profile is captured for this +# one-off smoke invocation (design's bazel-pure invocation list +# names build-wasm, build-pure, test-pure and test-bd only). bazel run --config=pure //cmd/bd:bd -- version bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ //internal/storage/embeddeddolt:embeddeddolt_test \ //internal/tracker:tracker_test \ --test_sharding_strategy=disabled \ '--test_arg=-test.run=^$' \ - --test_env=BEADS_TEST_SKIP=dolt`, + --test_env=BEADS_TEST_SKIP=dolt \ + --profile="$RUNNER_TEMP/bazel-profile-test-pure.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep-test-pure.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-test-pure.log.zst" \ + --experimental_build_event_upload_strategy=local`, "Run pure-Go cmd/bd test subset (--config=pure)": `set -euo pipefail bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} //cmd/bd:bd_test \ --test_sharding_strategy=disabled \ - "--test_arg=-test.run=$PURE_CMD_BD_TESTS" + "--test_arg=-test.run=$PURE_CMD_BD_TESTS" \ + --profile="$RUNNER_TEMP/bazel-profile-test-bd.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep-test-bd.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-test-bd.log.zst" \ + --experimental_build_event_upload_strategy=local n="$(grep -c ' 0 ))`, @@ -133,7 +146,12 @@ echo "pure cmd/bd subset: $n test cases" "bazel test //... --config=doltserver": `set -o pipefail start=$(date +%s) rc=0 -bazel test //... --config=doltserver ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? +bazel test //... --config=doltserver ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc"`, }, diff --git a/tools/bazel/ci_analytics_extract.py b/tools/bazel/ci_analytics_extract.py new file mode 100644 index 000000000..7f5d9bc3d --- /dev/null +++ b/tools/bazel/ci_analytics_extract.py @@ -0,0 +1,1276 @@ +#!/usr/bin/env python3 +"""Build a redacted `ci-analytics-summary` v1 JSON from a lane's Bazel outputs. + +Design: rbe-ci-bep-analytics-design.md, section 3. This file is a byte copy +across gascity and beads (tools/bazel/ci_analytics_extract.py in both): +stdlib Python 3.12+, like critpath.py. + +Inputs, per `--invocation STEP BEP [EXEC_LOG|-] [PROFILE|-]` (repeatable, at +most 8): + + - BEP JSON (`--build_event_json_file`): required. Timing, runner counts, + test results and summaries, network bytes. + - the compact execution log (`--execution_log_compact_file`, zstd-framed + protobuf): per-spawn mnemonic, runner, cache hit, target, SpawnMetrics. + "-" skips it (no per-mnemonic or per-cold data for that invocation). + - the JSON profile (`--profile`): "Fetching repository" wall time per + external repository. "-" skips it. + +Redaction is by construction (never read, by omission), by validation (every +emitted string is regex-checked against a whitelist and against the tripwire +individually, so one hostile value is dropped rather than losing the whole +document), and by a final tripwire regex over the serialized output as a +backstop. See REDACTION in this file's comments at each site, and the +design's section 3.4. + +Every line of BEP or exec log input is adversarial-input tolerant: a +malformed middle BEP line still drops the invocation (unchanged contract), +but a malformed individual event, protobuf entry or field degrades -- +skipping just that event/entry/field -- rather than crashing the process. +See ParseTimeout and the try/except boundaries around the exec-log and BEP +passes in build_invocation(). + +Exit codes: 0 written | 3 no BEP for any invocation (nothing written) | +4 tripwire (nothing written). + +Per-invocation `exec_log`/`profile` status values, for the S4 collector: + - "ok": present, readable, and parsed (possibly with individually + dropped/degraded entries -- that alone does not change this status). + - "missing": "-" was passed for that input; it was never read. + - "unreadable": the path couldn't be opened (missing file, permissions). + - "corrupt": opened, but the content was unparseable as a whole, or (for + the profile) a non-finite ts/dur value was seen and dropped. + - "timeout": PARSE_DEADLINE_S was exceeded partway through parsing; any + entries already parsed are kept. + - "too_large": the input exceeded MAX_EXEC_LOG_BYTES after decompression. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import math +import os +import re +import shutil +import subprocess +import sys +import time + +FORMAT = "ci-analytics-summary" +SCHEMA_VERSION = 1 +EXTRACTOR_VERSION = "1.0.0" + +MAX_INVOCATIONS = 8 +MAX_MNEMONICS = 200 +MAX_COLD = 100 +MAX_REPOS = 10 +MAX_TEST_LABELS = 5000 +MAX_TEST_RESULTS = 20000 # total across all labels, not per label +MAX_LABEL_LEN = 512 +MAX_EXEC_LOG_BYTES = 512 * 1024 * 1024 +PARSE_DEADLINE_S = 60.0 +DECOMPRESS_CHUNK = 65536 + +# How often (in entries/lines) the deadline is re-checked in the hot loops. +# Checking time.monotonic() on every iteration would be needlessly slow; +# checking too rarely defeats the point of a deadline. +DEADLINE_CHECK_EVERY = 2000 + +ZSTD_MAGIC = b"\x28\xb5\x2f\xfd" + +# REDACTION: every string this extractor emits is validated against one of +# these, or dropped. No raw path, hostname, URI or free-text field is ever +# written verbatim; the only free text read at all is the two buildToolLogs +# lines named below, and only through these patterns. +LABEL_RE = re.compile( + r"(?:@@?[A-Za-z0-9_.~+-]{0,100})?//(?:[A-Za-z0-9_.+=,@~-]+(?:/[A-Za-z0-9_.+=,@~-]+)*)?" + r":(?!/)[A-Za-z0-9_.+=,@~/-]{1,200}" +) +MNEMONIC_RE = re.compile(r"[A-Za-z][A-Za-z0-9_]{0,63}") +REPO_RE = re.compile(r"@@?[A-Za-z0-9_.~+-]{1,200}") +ENUM_RE = re.compile(r"[A-Z_]{1,40}") +# started.command: a Bazel subcommand name ("test", "build", "coverage", ...). +COMMAND_RE = re.compile(r"[a-z][a-z-]{0,31}") +# started.buildToolVersion: a release ("9.2.0") or a dev build string. +VERSION_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,63}") +# --invocation STEP, --lane, and matrix.lane values (bazel.yml): short, +# shell- and filename-safe identifiers, never free text. +LANE_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}") +# started.uuid / the BEP's own invocation id: a canonical (lowercase-or-not) +# UUID, nothing else. +UUID_RE = re.compile( + r"[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}" +) +# --mode: the rbe job's decide-mode output (design section 2). Checked as an +# exact set, not a regex. +MODE_CHOICES = frozenset({"remote", "fork-ro", "fork-rw", "cache", "local"}) + +# REDACTION: the tripwire. If this matches the serialized output, the +# extractor writes nothing and exits 4. It is also checked per-string at +# validation time (REDACTION below, in _validate()), so a single hostile +# label or mnemonic is dropped rather than taking down the whole document; +# this document-level check is the backstop for anything that still slips +# through, e.g. a free-text buildToolLogs line. The collector (design +# section 4.3) runs the same regex over the raw artifact text. +TRIPWIRE_RE = re.compile( + r"(?i)(://|/home/|/tmp/|/root/|/runner/|/etc/|-----BEGIN|\.ops\.gascity\.com|" + r"\b(?:\d{1,3}\.){3}\d{1,3}\b)" +) + +CRITICAL_PATH_RE = re.compile( + r"Critical Path: ([0-9.]+)s, Remote \(([0-9.]+)% of the time\): \[(.*)\]" +) +# Each bracketed component ("parse: 0.00%", "queue: 0.00%", ...); matched +# generically and then filtered to the keys this schema keeps. +CRITICAL_PATH_PCT_RE = re.compile(r"([A-Za-z]+): ([0-9.]+)%") +# The per-test critical-path line; the shard clause is optional so unsharded +# tests ("action 'Testing //pkg:t'", no "(shard N of M)") match too. +CRITICAL_PATH_TEST_RE = re.compile( + r"\s+([0-9.]+)s action 'Testing (//[^ ']+)(?: \(shard ([0-9]+) of ([0-9]+)\))?'" +) +DURATION_RE = re.compile(r"(-?[0-9]+(?:\.[0-9]+)?)s") + +# SpawnMetrics Durations (spawn.proto @ 9.2.0). Field 11 (input_bytes) is a +# varint, handled separately. +SPAWN_METRIC = { + 1: "total_ms", + 3: "network_ms", + 4: "fetch_ms", + 5: "queue_ms", + 6: "setup_ms", + 7: "upload_ms", + 8: "exec_ms", + 9: "outputs_ms", +} + +# Runner classes (design section 3.3). The raw runner name is never emitted. +_RUNNER_EXACT = { + "remote cache hit": "remote_cache_hit", + "disk cache hit": "disk_cache_hit", + "remote": "remote", +} +_RUNNER_LOCAL = { + "linux-sandbox", + "processwrapper-sandbox", + "darwin-sandbox", + "local", + "worker", + "standalone", + "sandboxed", + "windows-sandbox", + "docker", + "dynamic", +} + + +def runner_class(name: str) -> str: + if name in _RUNNER_EXACT: + return _RUNNER_EXACT[name] + if name in _RUNNER_LOCAL: + return "local" + return "other" + + +class Tripwire(Exception): + """Raised when the tripwire regex matches; caught at top level.""" + + +class ParseTimeout(Exception): + """Raised by read_bep()/spawns() when PARSE_DEADLINE_S is exceeded.""" + + +def _reject_constant(constant): # json.loads(parse_constant=...) + raise ValueError("non-finite JSON constant %r" % (constant,)) + + +def as_int(value, default=0): + """Decodes a proto3-JSON integer: Bazel writes int64 as a string and + int32 as a number; both decode here. Guards against OverflowError from + a huge/infinite float (e.g. a crafted 1e400 literal, or float('inf')).""" + if value is None: + return default + if isinstance(value, bool): + return int(value) + if isinstance(value, int): + return value + if isinstance(value, float): + try: + return int(value) + except (OverflowError, ValueError): + return default + if isinstance(value, str): + try: + return int(value) + except (ValueError, OverflowError): + return default + return default + + +def as_duration_ms(value): + """Decodes a proto3 Duration JSON string ("3.140856289s") to int ms.""" + if not isinstance(value, str): + return None + m = DURATION_RE.fullmatch(value) + if not m: + return None + try: + return round(float(m.group(1)) * 1000) + except (ValueError, OverflowError): + return None + + +def _safe_float(s, default=0.0): + try: + return float(s) + except (ValueError, OverflowError, TypeError): + return default + + +def _clamp_pct(value): + """Clamps a critical-path percentage to a finite [0, 100]. The regex + that extracts these values only ever captures digits and a single dot, + so float() on it can't itself produce inf/nan -- but this is cheap + insurance against a future, looser regex or a _safe_float() call site + that doesn't exist yet, and it's what keeps every *_pct value safe to + pass straight to json.dumps(..., allow_nan=False).""" + if not math.isfinite(value): + return 0.0 + return max(0.0, min(100.0, value)) + + +def _validate(s, pattern, max_len=None): + """REDACTION: the shared validator every valid_*() below calls. Checks + type, an optional length cap, the whitelist pattern (fullmatch: a + pattern ending in a bare `$` can still be satisfied by a string with a + trailing "\\n" under .match(), since `$` matches just before a trailing + newline -- fullmatch closes that gap), and the tripwire, individually. + A value that fails any of these is dropped (returns None); the caller + simply omits that one field/row rather than the whole document.""" + if not isinstance(s, str): + return None + if max_len is not None and len(s) > max_len: + return None + if not pattern.fullmatch(s): + return None + if TRIPWIRE_RE.search(s): + return None + return s + + +def valid_label(s): + return _validate(s, LABEL_RE, MAX_LABEL_LEN) + + +def valid_mnemonic(s): + return _validate(s, MNEMONIC_RE) + + +def valid_repo(s): + return _validate(s, REPO_RE) + + +def valid_enum(s): + return _validate(s, ENUM_RE) + + +def valid_command(s): + return _validate(s, COMMAND_RE) + + +def valid_version(s): + return _validate(s, VERSION_RE) + + +def valid_lane(s): + return _validate(s, LANE_RE) + + +def valid_uuid(s): + return _validate(s, UUID_RE) + + +# --- exec log: field-number protobuf decoding (spawn.proto @ 9.2.0) --- +# +# REDACTION: Spawn fields 7 target, 8 mnemonic, 9 exit code, 11 runner, 12 +# cache hit and 18 metrics are decoded; Invocation.id (field 4) is read only +# for the cross-check. Fields 1 (args), 2 (env_vars) and 3 (platform) are +# skipped by length, never decoded: this is the only place the exec log's +# argv and environment values could leak, and this code never looks at them. +# +# Every function here assumes nothing about its input's shape beyond what +# the length-prefixed wire format itself guarantees; a value that isn't the +# type a field is supposed to have (e.g. a string field re-encoded as a +# varint by a hostile or corrupt producer) is skipped, not force-cast. What +# this layer can't degrade from on its own (a truncated varint, an +# unsupported wire type, an absurd nesting) is left to raise; the caller +# (build_mnemonics_and_cold, from build_invocation) wraps the whole pass in +# try/except and degrades the invocation's exec_log status to "corrupt" or +# "timeout" instead of crashing the process. + + +def varint(b, p): + s = r = 0 + n = len(b) + while True: + if p >= n: + raise ValueError("truncated varint") + c = b[p] + p += 1 + r |= (c & 0x7F) << s + s += 7 + if not c & 0x80: + return r, p + if s > 63: + # Bounds the loop (a crafted run of 0xFF bytes with no + # terminator would otherwise grow `r` into an arbitrarily large + # Python int, making every subsequent shift/OR progressively + # more expensive -- an O(n^2) DoS, not just an eventual + # IndexError) and rejects anything wider than a 64-bit value. + raise ValueError("varint exceeds 64 bits") + + +def fields(b, deadline=None): + """Yields (field, wiretype, value); length-delimited values are slices, + never decoded here. + + A single field-rich submessage (e.g. a 40 MB Spawn made of millions of + tiny 2-byte unknown fields) can take many seconds to merely iterate, + even though spawns()'s own per-entry deadline check never sees it: that + check only fires once per ExecLogEntry, not once per field inside one. + deadline, when given, is re-checked every DEADLINE_CHECK_EVERY fields + here too, so a single oversized submessage at any nesting level can't + run past PARSE_DEADLINE_S on its own.""" + p = 0 + n = len(b) + count = 0 + while p < n: + count += 1 + if deadline is not None and count % DEADLINE_CHECK_EVERY == 0 and time.monotonic() > deadline: + raise ParseTimeout("protobuf field parse deadline exceeded") + k, p = varint(b, p) + f, t = k >> 3, k & 7 + if t == 0: + v, p = varint(b, p) + elif t == 2: + ln, p = varint(b, p) + v = b[p : p + ln] + p += ln + elif t == 1: + v = b[p : p + 8] + p += 8 + elif t == 5: + v = b[p : p + 4] + p += 4 + else: + raise ValueError("wire type %d" % t) + yield f, t, v + + +def dur_ms(v, deadline=None): + """Decodes a Duration submessage's bytes (fields 1 seconds, 2 nanos).""" + seconds = nanos = 0 + for f, t, val in fields(v, deadline): + if t != 0: + continue + if f == 1: + seconds = val + elif f == 2: + nanos = val + return seconds * 1000 + nanos // 1_000_000 + + +def spawns(raw, deadline): + """Yields (invocation_id, spawn-dict) for every ExecLogEntry.spawn in the + decompressed exec log stream. ExecLogEntry: 1 id, 2 invocation, 7 spawn; + the rest skipped. Every sub-field is type-checked before use: a hostile + or corrupt encoder can put any wire type on any field number, and a + string/submessage field that arrives as a varint (int, not bytes) is + simply skipped rather than passed to .decode()/fields().""" + p = 0 + n = len(raw) + inv = None + count = 0 + while p < n: + count += 1 + if count % DEADLINE_CHECK_EVERY == 0 and time.monotonic() > deadline: + raise ParseTimeout("exec log parse deadline exceeded") + ln, p = varint(raw, p) + entry = raw[p : p + ln] + p += ln + for f, _, v in fields(entry, deadline): + if f == 2 and isinstance(v, bytes): + inv = next( + ( + sv.decode("utf-8", "replace") + for sf, _, sv in fields(v, deadline) + if sf == 4 and isinstance(sv, bytes) + ), + inv, + ) + elif f == 7 and isinstance(v, bytes): + s = {"target": "", "mnemonic": "", "runner": "", "cache_hit": False, "m": {}} + for sf, st, sv in fields(v, deadline): # 1 args, 2 env_vars, 3 platform: never decoded + if sf == 7 and isinstance(sv, bytes): + s["target"] = sv.decode("utf-8", "replace") + elif sf == 8 and isinstance(sv, bytes): + s["mnemonic"] = sv.decode("utf-8", "replace") + elif sf == 11 and isinstance(sv, bytes): + s["runner"] = sv.decode("utf-8", "replace") + elif sf == 12: + s["cache_hit"] = bool(sv) + elif sf == 18 and isinstance(sv, bytes): + for mf, mt, mv in fields(sv, deadline): + if mf in SPAWN_METRIC and mt == 2 and isinstance(mv, bytes): + s["m"][SPAWN_METRIC[mf]] = dur_ms(mv, deadline) + elif mf == 11 and mt == 0: + s["m"]["input_bytes"] = mv + yield inv, s + + +def load_exec_log(path, deadline): + """Returns (decompressed bytes or None, status) where status is one of + "ok", "unreadable", "too_large"; None data means no per-spawn data for + this invocation (BEP-level data is still emitted). + + The compressed file's size is stat-capped before it is ever opened, and + decompression itself runs under a Popen whose stdout is read in bounded + chunks: at most MAX_EXEC_LOG_BYTES + 1 bytes are ever read before the + child is killed, so a zstd decompression bomb can't grow this process's + memory past that bound no matter how small the compressed input is. + --long=31 (which lets zstd negotiate up to a 2 GiB window) is + deliberately not passed, for the same reason. + """ + try: + st = os.stat(path) + except OSError: + return None, "unreadable" + if st.st_size > MAX_EXEC_LOG_BYTES: + return None, "too_large" + try: + with open(path, "rb") as fh: + magic = fh.read(4) + except OSError: + return None, "unreadable" + if magic != ZSTD_MAGIC: + try: + with open(path, "rb") as fh: + data = fh.read() + except OSError: + return None, "unreadable" + if len(data) > MAX_EXEC_LOG_BYTES: + return None, "too_large" + return data, "ok" + + zstd = shutil.which("zstd") + if zstd is None: + return None, "unreadable" + try: + in_fh = open(path, "rb") + except OSError: + return None, "unreadable" + try: + proc = subprocess.Popen( + [zstd, "-dc"], + stdin=in_fh, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + except OSError: + return None, "unreadable" + finally: + in_fh.close() # Popen dup'd it; closing our copy doesn't affect the child. + + chunks = [] + total = 0 + too_large = False + try: + try: + while True: + if time.monotonic() > deadline: + proc.kill() + proc.wait() + return None, "unreadable" + chunk = proc.stdout.read(DECOMPRESS_CHUNK) + if not chunk: + break + total += len(chunk) + if total > MAX_EXEC_LOG_BYTES: + too_large = True + proc.kill() + break + chunks.append(chunk) + proc.wait(timeout=max(1.0, deadline - time.monotonic())) + except (OSError, subprocess.SubprocessError): + try: + proc.kill() + except OSError: + pass + return None, "unreadable" + finally: + proc.stdout.close() + + if too_large: + return None, "too_large" + if proc.returncode != 0: + return None, "unreadable" + return b"".join(chunks), "ok" + + +# --- BEP: newline-delimited JSON --- + + +def read_bep(path, deadline): + """Reads the BEP JSON stream. Returns (events, truncated). A malformed + last line sets truncated; a malformed middle line raises ValueError (the + whole invocation is dropped, the same contract as gascity bep.go).""" + events = [] + pending_err = None + count = 0 + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + count += 1 + if count % DEADLINE_CHECK_EVERY == 0 and time.monotonic() > deadline: + raise ParseTimeout("BEP parse deadline exceeded") + line = line.strip() + if not line: + continue + if pending_err is not None: + raise ValueError(pending_err) + try: + events.append(json.loads(line, parse_constant=_reject_constant)) + except ValueError as exc: + pending_err = str(exc) + return events, pending_err is not None + + +def iter_repo_fetch(profile_path, deadline): + """Streams the JSON profile line by line; only lines mentioning + "Fetching repository" are parsed (design section 3.5's cap). Computes + the wall-clock union of each name's intervals (not just a min-start/ + max-end span: two disjoint fetches of the same repo should not count the + gap between them). Returns (rows, status), status one of "ok", + "unreadable", "corrupt" (a non-finite ts/dur -- Infinity, NaN, or a sum + that overflows to inf -- was seen and dropped; any other valid rows + parsed are still returned).""" + intervals = {} # name -> sorted list of disjoint, merged [start, end) + corrupt = False + try: + fh = open(profile_path, "r", encoding="utf-8", errors="replace") + except OSError: + return [], "unreadable" + with fh: + for line in fh: + if time.monotonic() > deadline: + break + if "Fetching repository" not in line: + continue + stripped = line.rstrip(",\n") + try: + # Default parse_constant (not _reject_constant): a crafted + # "ts": Infinity/NaN literal must reach the isfinite() + # check below and flip this profile's status to "corrupt", + # not raise out of json.loads() and get silently skipped. + ev = json.loads(stripped) + except ValueError: + continue + if not isinstance(ev, dict) or ev.get("cat") != "Fetching repository": + continue + name = ev.get("name") + ts = ev.get("ts") + dur = ev.get("dur") + if ( + not isinstance(name, str) + or not isinstance(ts, (int, float)) + or not isinstance(dur, (int, float)) + ): + continue + # A crafted ts/dur (1e999 parses as inf via _reject_constant's + # sibling path below; ts=1e308 + dur=1e308 overflows a finite + # sum to inf without raising) must degrade this invocation's + # profile status to "corrupt", not crash or silently produce an + # infinite/NaN interval that a naive sum() would then also + # produce as a reported "ms" value. + try: + start, end = ts, ts + dur + if not (math.isfinite(start) and math.isfinite(end)): + corrupt = True + continue + except (OverflowError, ValueError, TypeError): + corrupt = True + continue + merged = intervals.setdefault(name, []) + _merge_interval(merged, start, end) + out = [] + for name, merged in intervals.items(): + repo = valid_repo(name) + if repo is None: + continue + total_us = sum(e - s for s, e in merged) + out.append({"repo": repo, "ms": round(total_us / 1000.0)}) + out.sort(key=lambda r: r["ms"], reverse=True) + return out[:MAX_REPOS], ("corrupt" if corrupt else "ok") + + +def _merge_interval(merged, start, end): + """Inserts [start, end) into `merged`, a list of disjoint intervals kept + sorted by start, merging with any overlapping/adjacent neighbors so the + list always represents the true union.""" + out = [] + placed = False + for s, e in merged: + if end < s: + if not placed: + out.append((start, end)) + placed = True + out.append((s, e)) + elif start > e: + out.append((s, e)) + else: + start, end = min(start, s), max(end, e) + if not placed: + out.append((start, end)) + out.sort() + merged[:] = out + + +# --- per-invocation BEP extraction --- + + +def _empty_critical_path(): + return { + "remote_pct": 0.0, + "queue_pct": 0.0, + "fetch_pct": 0.0, + "process_pct": 0.0, + "upload_pct": 0.0, + "setup_pct": 0.0, + "network_pct": 0.0, + # Always present (the S4 contract: a stable, always-complete key + # set), even when the critical path text has no per-test line at + # all (a pure-build invocation, or a critpath_bad_float-style + # degrade): "" / 0, never an absent key the collector has to guess + # the type of. + "test_label": "", + "test_shard": 0, + "test_ms": 0, + } + + +def build_timing(events): + timing = { + "elapsed_ms": 0, + "wall_ms": 0, + "cpu_ms": 0, + "analysis_ms": 0, + "execution_ms": 0, + "actions_start_ms": 0, + "critical_path_ms": 0, + } + started = None + finished = None + # Always a full key set (design/S4 contract): a lane with no critical + # path text still reports every *_pct at 0.0, rather than an empty dict. + critical_path = _empty_critical_path() + actions = { + "created": 0, + "executed": 0, + "remote_cache_hit": 0, + "disk_cache_hit": 0, + "remote": 0, + "local": 0, + "internal": 0, + "action_cache_hits": 0, + "action_cache_misses": 0, + } + graph = {"packages_loaded": 0, "targets_configured": 0} + net = {"bytes_recv": 0, "bytes_sent": 0} + + for ev in events: + # Tolerates a BEP line that decoded to a list, a string, or any + # other non-dict JSON value (malformed middle lines already raise + # in read_bep(); this is for individually-wrong-shaped lines that + # parsed fine as JSON but aren't the object BEP lines always are). + if not isinstance(ev, dict): + continue + try: + if "started" in ev: + s = ev["started"] + if isinstance(s, dict): + started = { + "invocation_id": valid_uuid(s.get("uuid", "")) or "", + "command": valid_command(s.get("command", "")) or "unknown", + "bazel_version": valid_version(s.get("buildToolVersion", "")) or "unknown", + "started_ms": as_int(s.get("startTimeMillis")), + } + if "finished" in ev: + fin = ev["finished"] + if isinstance(fin, dict): + exit_code = fin.get("exitCode") + if not isinstance(exit_code, dict): + exit_code = {} + finished = { + "exit_code": as_int(exit_code.get("code")), + "exit_name": exit_code.get("name", ""), + "finished_ms": as_int(fin.get("finishTimeMillis")), + } + if "buildToolLogs" in ev: + btl = ev["buildToolLogs"] + logs = btl.get("log", []) if isinstance(btl, dict) else [] + for log in logs or []: + if not isinstance(log, dict): + continue + name = log.get("name") + contents = log.get("contents") + if name == "elapsed time" and contents: + try: + timing["elapsed_ms"] = round(float(base64.b64decode(contents)) * 1000) + except (ValueError, TypeError, OverflowError): + pass + elif name == "critical path" and contents: + try: + text = base64.b64decode(contents).decode("utf-8", "replace") + except (ValueError, TypeError): + text = "" + first_line = text.splitlines()[0] if text else "" + m = CRITICAL_PATH_RE.fullmatch(first_line) + if m: + cp = dict(critical_path) + cp["remote_pct"] = _clamp_pct(_safe_float(m.group(2))) + for key, val in CRITICAL_PATH_PCT_RE.findall(m.group(3)): + field = "%s_pct" % key + if field in cp: + cp[field] = _clamp_pct(_safe_float(val)) + critical_path = cp + for line in text.splitlines()[1:]: + tm = CRITICAL_PATH_TEST_RE.fullmatch(line) + if tm: + label = valid_label(tm.group(2)) + if label is not None: + critical_path["test_label"] = label + critical_path["test_shard"] = ( + as_int(tm.group(3)) if tm.group(3) else 0 + ) + critical_path["test_ms"] = round( + _safe_float(tm.group(1)) * 1000 + ) + break + if "buildMetrics" in ev: + bm = ev["buildMetrics"] + if not isinstance(bm, dict): + continue + tm = bm.get("timingMetrics") + if not isinstance(tm, dict): + tm = {} + for key, dst in ( + ("cpuTimeInMs", "cpu_ms"), + ("wallTimeInMs", "wall_ms"), + ("analysisPhaseTimeInMs", "analysis_ms"), + ("executionPhaseTimeInMs", "execution_ms"), + ("actionsExecutionStartInMs", "actions_start_ms"), + ): + if key in tm: + timing[dst] = as_int(tm.get(key)) + # Bazel 9.2.0 sends criticalPathTime as a Duration string + # ("0.203047257s"), not criticalPathTimeInMs: as_int() on + # that string always returned the default 0, silently. + # Parse the Duration first; fall back to a legacy/future + # criticalPathTimeInMs integer field if present instead. + cpt_ms = as_duration_ms(tm.get("criticalPathTime")) + if cpt_ms is None and "criticalPathTimeInMs" in tm: + cpt_ms = as_int(tm.get("criticalPathTimeInMs")) + if cpt_ms is not None: + timing["critical_path_ms"] = cpt_ms + + asum = bm.get("actionSummary") + if not isinstance(asum, dict): + asum = {} + if "actionsCreated" in asum: + actions["created"] = as_int(asum.get("actionsCreated")) + if "actionsExecuted" in asum: + actions["executed"] = as_int(asum.get("actionsExecuted")) + local_count = 0 + for rc in asum.get("runnerCount", []) or []: + if not isinstance(rc, dict): + continue + name = rc.get("name", "") + count = as_int(rc.get("count")) + if rc.get("execKind") == "Local": + local_count += count + continue + if name == "remote cache hit": + actions["remote_cache_hit"] += count + elif name == "disk cache hit": + actions["disk_cache_hit"] += count + elif name == "remote": + actions["remote"] += count + elif name == "internal": + actions["internal"] += count + actions["local"] = local_count + acs = asum.get("actionCacheStatistics") + if isinstance(acs, dict): + actions["action_cache_hits"] = as_int(acs.get("hits")) + actions["action_cache_misses"] = as_int(acs.get("misses")) + + pm = bm.get("packageMetrics") + if isinstance(pm, dict): + graph["packages_loaded"] = as_int(pm.get("packagesLoaded")) + tgm = bm.get("targetMetrics") + if isinstance(tgm, dict): + graph["targets_configured"] = as_int(tgm.get("targetsConfigured")) + nmw = bm.get("networkMetrics") + nm = nmw.get("systemNetworkStats") if isinstance(nmw, dict) else None + if isinstance(nm, dict): + net["bytes_recv"] = as_int(nm.get("bytesRecv")) + net["bytes_sent"] = as_int(nm.get("bytesSent")) + except Exception: + # Degrade to a partial BEP: one malformed event (e.g. "started" + # or "testResult" present but holding a non-dict value) is + # skipped; every other event in this stream is still processed. + continue + + return started, finished, timing, critical_path, actions, graph, net + + +def build_action_data(events): + """BEP buildMetrics.actionSummary.actionData[]: per-mnemonic executed, + created, userTime, systemTime. Top 20 only (Bazel's own cap).""" + data = {} + for ev in events: + if not isinstance(ev, dict): + continue + try: + bm = ev.get("buildMetrics") + if not isinstance(bm, dict): + continue + asum = bm.get("actionSummary") + if not isinstance(asum, dict): + continue + for ad in asum.get("actionData", []) or []: + if not isinstance(ad, dict): + continue + mnemonic = valid_mnemonic(ad.get("mnemonic", "")) + if mnemonic is None: + continue + d = data.setdefault( + mnemonic, {"bep_executed": 0, "bep_created": 0, "user_ms": 0, "system_ms": 0} + ) + d["bep_executed"] += as_int(ad.get("actionsExecuted")) + d["bep_created"] += as_int(ad.get("actionsCreated")) + d["user_ms"] += as_duration_ms(ad.get("userTime")) or 0 + d["system_ms"] += as_duration_ms(ad.get("systemTime")) or 0 + except Exception: + continue + return data + + +def build_tests(events): + """testResult + testSummary, grouped by label. MAX_TEST_RESULTS is a + total across every label (the S4 contract), not a per-label cap.""" + tests = {} # label -> {summary, results: []} + truncated = 0 + total_results = 0 + for ev in events: + if not isinstance(ev, dict): + continue + try: + ident = ev.get("id") + if not isinstance(ident, dict): + ident = {} + tr = ev.get("testResult") + if tr is not None and isinstance(tr, dict): + tid = ident.get("testResult") + if not isinstance(tid, dict): + tid = {} + label = valid_label(tid.get("label", "")) + if label is None: + continue + entry = tests.setdefault(label, {"summary": None, "results": []}) + if total_results >= MAX_TEST_RESULTS: + truncated += 1 + continue + status = valid_enum(tr.get("status", "")) or "UNKNOWN" + execinfo = tr.get("executionInfo") + if not isinstance(execinfo, dict): + execinfo = {} + strategy = execinfo.get("strategy", "") + cached_locally = bool(tr.get("cachedLocally")) + runner = "local_cache" if cached_locally else runner_class(strategy) + duration_ms = as_duration_ms(tr.get("testAttemptDuration")) + if duration_ms is None: + duration_ms = as_int(tr.get("testAttemptDurationMillis")) + queue_ms = fetch_ms = exec_ms = 0 + # Per-test timing lives under testResult.executionInfo. + # timingBreakdown, not a top-level testResult.timingBreakdown + # (which doesn't exist at all) -- confirmed against a real + # Bazel 9.2 fixture. + tbd = execinfo.get("timingBreakdown") + children = tbd.get("child", []) if isinstance(tbd, dict) else [] + for child in children or []: + if not isinstance(child, dict): + continue + name = child.get("name") + ms = as_duration_ms(child.get("time")) + if ms is None: + continue + if name == "queueTime": + queue_ms = ms + elif name == "fetchTime": + fetch_ms = ms + elif name == "executionWallTime": + exec_ms = ms + entry["results"].append( + { + "shard": as_int(tid.get("shard")), + "run": as_int(tid.get("run")), + "attempt": as_int(tid.get("attempt")), + "status": status, + "runner": runner, + "duration_ms": duration_ms, + "queue_ms": queue_ms, + "fetch_ms": fetch_ms, + "exec_ms": exec_ms, + } + ) + total_results += 1 + ts = ev.get("testSummary") + if ts is not None and isinstance(ts, dict): + tid = ident.get("testSummary") + if not isinstance(tid, dict): + tid = {} + label = valid_label(tid.get("label", "")) + if label is None: + continue + entry = tests.setdefault(label, {"summary": None, "results": []}) + entry["summary"] = { + "overall": valid_enum(ts.get("overallStatus", "")) or "UNKNOWN", + "shards": as_int(ts.get("shardCount")) or 1, + "attempts": as_int(ts.get("attemptCount")) or 1, + "cached": as_int(ts.get("totalNumCached")), + } + except Exception: + continue + + labels = sorted(tests.keys()) + tests_truncated = 0 + if len(labels) > MAX_TEST_LABELS: + tests_truncated = len(labels) - MAX_TEST_LABELS + labels = labels[:MAX_TEST_LABELS] + out = [] + for label in labels: + entry = tests[label] + summary = entry["summary"] or { + "overall": "UNKNOWN", + "shards": 1, + "attempts": 1, + "cached": 0, + } + out.append( + { + "label": label, + "overall": summary["overall"], + "shards": summary["shards"], + "attempts": summary["attempts"], + "cached": summary["cached"], + "results": entry["results"], + } + ) + return out, tests_truncated + truncated + + +def build_mnemonics_and_cold(started_uuid, exec_log_data, deadline): + """Groups exec-log spawns by mnemonic (mnemonics[]) and finds the + cache-missing ones by (mnemonic, target) (cold[]).""" + by_mnemonic = {} + cold = {} + matched_invocation = False + if exec_log_data is None: + return {}, {}, False + for inv, spawn in spawns(exec_log_data, deadline): + if inv is not None: + if started_uuid and inv != started_uuid: + continue + matched_invocation = True + mnemonic = valid_mnemonic(spawn["mnemonic"]) + if mnemonic is None: + continue + cls = runner_class(spawn["runner"]) + m = spawn["m"] + d = by_mnemonic.setdefault( + mnemonic, + { + "spawns": 0, + "remote_cache_hit": 0, + "disk_cache_hit": 0, + "remote": 0, + "local": 0, + "other": 0, + "total_ms": 0, + "remote_exec_ms": 0, + "remote_queue_ms": 0, + "remote_queue_max_ms": 0, + "remote_setup_ms": 0, + "local_exec_ms": 0, + "fetch_ms": 0, + "upload_ms": 0, + "input_bytes": 0, + }, + ) + d["spawns"] += 1 + if cls in ("remote_cache_hit", "disk_cache_hit", "remote", "local"): + d[cls] += 1 + else: + d["other"] += 1 + d["total_ms"] += m.get("total_ms", 0) + d["fetch_ms"] += m.get("fetch_ms", 0) + d["upload_ms"] += m.get("upload_ms", 0) + d["input_bytes"] += m.get("input_bytes", 0) + if cls == "remote": + d["remote_exec_ms"] += m.get("exec_ms", 0) + d["remote_queue_ms"] += m.get("queue_ms", 0) + d["remote_queue_max_ms"] = max(d["remote_queue_max_ms"], m.get("queue_ms", 0)) + d["remote_setup_ms"] += m.get("setup_ms", 0) + elif cls == "local": + d["local_exec_ms"] += m.get("exec_ms", 0) + + if cls in ("remote", "local"): # cache misses + target = valid_label(spawn["target"]) + if target is not None: + key = (mnemonic, target, cls) + cd = cold.setdefault( + key, {"mnemonic": mnemonic, "target": target, "runner": cls, "spawns": 0, "total_ms": 0, "exec_ms": 0, "queue_ms": 0} + ) + cd["spawns"] += 1 + cd["total_ms"] += m.get("total_ms", 0) + cd["exec_ms"] += m.get("exec_ms", 0) + cd["queue_ms"] += m.get("queue_ms", 0) + return by_mnemonic, cold, matched_invocation + + +def finalize_mnemonics(by_mnemonic, action_data): + truncated = 0 + names = sorted(by_mnemonic.keys()) + if len(names) > MAX_MNEMONICS: + truncated = len(names) - MAX_MNEMONICS + names = sorted(by_mnemonic.keys(), key=lambda n: by_mnemonic[n]["total_ms"], reverse=True)[:MAX_MNEMONICS] + names.sort() + out = [] + for name in names: + d = dict(by_mnemonic[name]) + d["mnemonic"] = name + extra = action_data.get(name) + if extra: + d.update(extra) + else: + d.setdefault("bep_executed", 0) + d.setdefault("bep_created", 0) + d.setdefault("user_ms", 0) + d.setdefault("system_ms", 0) + out.append(d) + return out, truncated + + +def finalize_cold(cold): + rows = list(cold.values()) + rows.sort(key=lambda r: r["total_ms"], reverse=True) + truncated = max(0, len(rows) - MAX_COLD) + return rows[:MAX_COLD], truncated + + +def build_invocation(step, bep_path, exec_log_path, profile_path, deadline): + try: + events, bep_truncated = read_bep(bep_path, deadline) + except OSError: + return None + except ValueError: + # A malformed non-final line: drop this invocation entirely. + return None + except ParseTimeout: + return None + + started, finished, timing, critical_path, actions, graph, net = build_timing(events) + if started is None: + return None + + # "-" (or omitting the argument) means "missing": there was never a file + # to read. That's different from "unreadable" (a path was given but + # couldn't be opened/decoded) or "corrupt"/"timeout" (opened, but the + # contents didn't decode, or took too long). + exec_log_status = "missing" + exec_log_data = None + if exec_log_path is not None: + exec_log_data, exec_log_status = load_exec_log(exec_log_path, deadline) + + mnemonics_raw, cold_raw = {}, {} + if exec_log_data is not None: + try: + mnemonics_raw, cold_raw, matched = build_mnemonics_and_cold( + started.get("invocation_id", ""), exec_log_data, deadline + ) + except ParseTimeout: + exec_log_status = "timeout" + except Exception: + # Every crash this layer can't itself degrade from (a truncated + # varint, an unsupported wire type, a sub-message that's the + # wrong shape in a way isinstance() checks upstream didn't + # anticipate) lands here: the invocation still gets its BEP-only + # data, just no per-mnemonic/cold rows. + exec_log_status = "corrupt" + else: + if started.get("invocation_id") and not matched: + exec_log_status = "mismatch" + mnemonics_raw, cold_raw = {}, {} + + action_data = build_action_data(events) + mnemonics, mnemonics_truncated = finalize_mnemonics(mnemonics_raw, action_data) + cold, cold_truncated = finalize_cold(cold_raw) + + repo_fetch = [] + profile_status = "missing" + if profile_path is not None: + repo_fetch, profile_status = iter_repo_fetch(profile_path, deadline) + + tests, tests_truncated = build_tests(events) + + record = { + "step": step, + "invocation_id": started["invocation_id"], + "command": started["command"], + "bazel_version": started["bazel_version"], + "started_ms": started["started_ms"], + "finished_ms": finished["finished_ms"] if finished else 0, + # No finished event at all (the lane crashed/was killed mid-build): + # -1, never the misleadingly-successful-looking default of 0. + "exit_code": finished["exit_code"] if finished else -1, + "exit_name": valid_enum(finished["exit_name"]) if finished else None, + "bep_truncated": bep_truncated, + "exec_log": exec_log_status, + "profile": profile_status, + "timing": timing, + "critical_path": critical_path, + "actions": actions, + "graph": graph, + "net": net, + "mnemonics": mnemonics, + "mnemonics_truncated": mnemonics_truncated, + "cold": cold, + "cold_truncated": cold_truncated, + "repo_fetch": repo_fetch, + "tests": tests, + "tests_truncated": tests_truncated, + } + if record["exit_name"] is None: + record["exit_name"] = "UNKNOWN" + return record + + +def build_summary(lane, mode, check_run_id, pr_hint, invocation_args, deadline): + invocations = [] + for step, bep, exec_log, profile in invocation_args: + record = build_invocation(step, bep, exec_log, profile, deadline) + if record is not None: + invocations.append(record) + if not invocations: + return None + return { + "format": FORMAT, + "version": SCHEMA_VERSION, + "extractor": EXTRACTOR_VERSION, + "lane": lane, + "mode": mode, + "check_run_id": as_int(check_run_id), + "pr_hint": as_int(pr_hint), + "invocations": invocations, + } + + +def parse_args(argv): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--lane", required=True) + ap.add_argument("--mode", required=True) + ap.add_argument("--check-run-id", required=True) + ap.add_argument("--pr-hint", required=True) + ap.add_argument("--out", required=True) + ap.add_argument( + "--invocation", + nargs="+", + action="append", + default=[], + metavar=("STEP BEP EXEC_LOG PROFILE"), + ) + args = ap.parse_args(argv) + + # REDACTION: --lane, --mode and every --invocation STEP are written + # straight into the output JSON (lane, mode, invocations[].step). They + # come from workflow env/matrix values, not free text, but nothing + # enforced that before this CLI parsed them -- a hostile or just wrong + # caller could put an auth-header-shaped string in --lane and have it + # land in the artifact verbatim. Validate eagerly, before any file I/O. + if valid_lane(args.lane) is None: + ap.error("--lane must match %s" % LANE_RE.pattern) + if args.mode not in MODE_CHOICES: + ap.error("--mode must be one of: %s" % ", ".join(sorted(MODE_CHOICES))) + + invocations = [] + for inv in args.invocation: + if len(inv) < 2 or len(inv) > 4: + ap.error("--invocation takes STEP BEP [EXEC_LOG|-] [PROFILE|-]") + step = inv[0] + if valid_lane(step) is None: + ap.error("--invocation STEP must match %s" % LANE_RE.pattern) + bep = inv[1] + exec_log = inv[2] if len(inv) > 2 and inv[2] != "-" else None + profile = inv[3] if len(inv) > 3 and inv[3] != "-" else None + invocations.append((step, bep, exec_log, profile)) + if len(invocations) > MAX_INVOCATIONS: + ap.error("at most %d --invocation entries" % MAX_INVOCATIONS) + if not invocations: + ap.error("at least one --invocation is required") + args.invocations = invocations + return args + + +def main(argv=None): + args = parse_args(argv if argv is not None else sys.argv[1:]) + deadline = time.monotonic() + PARSE_DEADLINE_S + try: + summary = build_summary( + args.lane, args.mode, args.check_run_id, args.pr_hint, args.invocations, deadline + ) + except Tripwire: + sys.stderr.write("::error::ci_analytics_extract: tripwire matched; nothing written\n") + return 4 + + if summary is None: + sys.stderr.write("::error::ci_analytics_extract: no BEP for any invocation; nothing written\n") + return 3 + + try: + # allow_nan=False: a backstop against ever writing invalid JSON + # (bare Infinity/-Infinity/NaN) should a non-finite number reach + # this point despite as_int()/as_duration_ms()/_clamp_pct() -- every + # numeric field this extractor emits is already supposed to be + # finite by the time it gets here. + text = json.dumps(summary, sort_keys=True, separators=(",", ":"), allow_nan=False) + except ValueError: + sys.stderr.write( + "::error::ci_analytics_extract: a non-finite number reached serialization; nothing written\n" + ) + return 4 + if TRIPWIRE_RE.search(text): + sys.stderr.write("::error::ci_analytics_extract: tripwire matched serialized output; nothing written\n") + return 4 + + out_dir = os.path.dirname(args.out) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + tmp_path = args.out + ".tmp" + with open(tmp_path, "w", encoding="utf-8") as fh: + fh.write(text) + os.replace(tmp_path, args.out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/ci_analytics_extract_test.py b/tools/bazel/ci_analytics_extract_test.py new file mode 100644 index 000000000..dc7de9205 --- /dev/null +++ b/tools/bazel/ci_analytics_extract_test.py @@ -0,0 +1,1429 @@ +"""Tests for ci_analytics_extract.py. + +Three fixture kinds, per the design doc (section 3.5): + + - testdata/ci_analytics/real/*: genuine Bazel 9.2.0 output from an + isolated scratch workspace (tools/bazel/testdata/ci_analytics/regen.sh), + with no remote flags and no ambient environment (env -i). The BEP is + filtered down to only the fields the extractor reads before being + committed; the exec log is committed decompressed (*.binpb) so these + tests don't need host `zstd`. Exercises the real BEP/exec-log/profile + shapes, including a failed test (flaky_test) and, in the post-clean + pair, a disk_cache_hit majority with one local re-execution. + - a synthetic remote fixture, built here with a small protobuf encoder, + for the `remote` and `remote cache hit` runner classes this sandbox has + no RBE endpoint to produce for real. + - testdata/ci_analytics/poisoned-bep.json: a BEP file whose every + extractor-visible field (including a couple of hostile labels) carries + an embedded secret or leaky value. The summary built from it must + contain none of them. +""" + +from __future__ import annotations + +import base64 +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) +import ci_analytics_extract as extract # noqa: E402 + +TESTDATA = os.path.join(os.path.dirname(__file__), "testdata", "ci_analytics") +REAL = os.path.join(TESTDATA, "real") + + +def _deadline(seconds=30.0): + return time.monotonic() + seconds + + +# --- a tiny protobuf encoder, for the synthetic remote fixture and the +# adversarial exec-log tests below --- + + +def _varint(n: int) -> bytes: + out = bytearray() + while True: + b = n & 0x7F + n >>= 7 + if n: + out.append(b | 0x80) + else: + out.append(b) + return bytes(out) + + +def _tag(field: int, wire: int) -> bytes: + return _varint((field << 3) | wire) + + +def _varint_field(field: int, n: int) -> bytes: + return _tag(field, 0) + _varint(n) + + +def _bytes_field(field: int, b: bytes) -> bytes: + return _tag(field, 2) + _varint(len(b)) + b + + +def _str_field(field: int, s: str) -> bytes: + return _bytes_field(field, s.encode("utf-8")) + + +def _duration_bytes(ms: int) -> bytes: + seconds, nanos = divmod(ms, 1000) + return _varint_field(1, seconds) + _varint_field(2, nanos * 1_000_000) + + +def _spawn_metrics(total_ms=0, queue_ms=0, exec_ms=0, input_bytes=0) -> bytes: + body = b"" + if total_ms: + body += _bytes_field(1, _duration_bytes(total_ms)) + if queue_ms: + body += _bytes_field(5, _duration_bytes(queue_ms)) + if exec_ms: + body += _bytes_field(8, _duration_bytes(exec_ms)) + if input_bytes: + body += _varint_field(11, input_bytes) + return body + + +def _spawn(target, mnemonic, runner, cache_hit, metrics, args=(), env=()) -> bytes: + body = b"" + for a in args: # field 1: args. Never decoded by the extractor. + body += _str_field(1, a) + for k, v in env: # field 2: env_vars. Never decoded by the extractor. + body += _bytes_field(2, _str_field(1, k) + _str_field(2, v)) + body += _str_field(7, target) + body += _str_field(8, mnemonic) + body += _str_field(11, runner) + body += _varint_field(12, 1 if cache_hit else 0) + body += _bytes_field(18, metrics) + return body + + +def _length_prefixed(entry: bytes) -> bytes: + return _varint(len(entry)) + entry + + +def _entry_with_spawn(spawn_body: bytes) -> bytes: + entry = _bytes_field(7, spawn_body) + return _length_prefixed(entry) + + +def _invocation_entry(uuid: str) -> bytes: + return _length_prefixed(_bytes_field(2, _str_field(4, uuid))) + + +def synthetic_exec_log(uuid: str) -> bytes: + """A compact exec log (uncompressed; load_exec_log reads it verbatim + when it does not start with the zstd magic) with an invocation entry + then two spawns: a `remote` miss with queue time, and a `remote cache + hit`.""" + out = _invocation_entry(uuid) + out += _entry_with_spawn( + _spawn( + "//:synth_remote", + "CppCompile", + "remote", + False, + _spawn_metrics(total_ms=900, queue_ms=250, exec_ms=500), + ) + ) + out += _entry_with_spawn( + _spawn( + "//:synth_cached", + "CppCompile", + "remote cache hit", + True, + _spawn_metrics(total_ms=20), + ) + ) + return out + + +def synthetic_bep(uuid: str) -> str: + events = [ + { + "id": {"started": {}}, + "started": { + "uuid": uuid, + "startTimeMillis": "1700000000000", + "buildToolVersion": "9.2.0", + "command": "build", + }, + }, + { + "id": {"buildMetrics": {}}, + "buildMetrics": { + "timingMetrics": { + "cpuTimeInMs": "900", + "wallTimeInMs": "950", + "criticalPathTime": "0.900s", + }, + "actionSummary": { + "actionsCreated": "2", + "actionsExecuted": "2", + "runnerCount": [ + {"name": "remote", "count": "1", "execKind": "Remote"}, + {"name": "remote cache hit", "count": "1", "execKind": "Remote"}, + ], + "actionData": [ + { + "mnemonic": "CppCompile", + "actionsExecuted": "2", + "actionsCreated": "2", + "userTime": "0.400s", + "systemTime": "0.100s", + } + ], + }, + }, + }, + { + "id": {"finished": {}}, + "finished": {"exitCode": {"code": 0, "name": "SUCCESS"}, "finishTimeMillis": "1700000001000"}, + }, + ] + return "\n".join(json.dumps(e) for e in events) + "\n" + + +class ExtractorHelperTests(unittest.TestCase): + def test_varint_roundtrip(self): + for n in (0, 1, 127, 128, 300, 2**32, 2**40 + 7): + encoded = _varint(n) + got, p = extract.varint(encoded, 0) + self.assertEqual(got, n) + self.assertEqual(p, len(encoded)) + + def test_varint_rejects_truncated_input(self): + with self.assertRaises(ValueError): + extract.varint(b"\x80", 0) # continuation bit set, nothing follows + + def test_varint_rejects_overlong_input(self): + # No terminator within 64 bits: must raise promptly, not loop while + # accumulating an ever-larger Python int (the quadratic-blowup DoS + # a reviewer's adversarial script reproduced against the + # unbounded version of this function). + start = time.monotonic() + with self.assertRaises(ValueError): + extract.varint(b"\xff" * 160000 + b"\x01", 0) + self.assertLess(time.monotonic() - start, 1.0) + + def test_valid_label(self): + self.assertEqual(extract.valid_label("//foo/bar:baz"), "//foo/bar:baz") + self.assertEqual(extract.valid_label("@@rules_cc+//cc:defs"), "@@rules_cc+//cc:defs") + self.assertIsNone(extract.valid_label("//foo; rm -rf /")) + self.assertIsNone(extract.valid_label("http://evil.example/x")) + + def test_valid_label_rejects_trailing_newline(self): + # fullmatch(), not match() with a trailing "$": "$" matches just + # before a trailing newline under match(), letting a smuggled "\n" + # through every validator. + self.assertIsNone(extract.valid_label("//pkg:name\n")) + self.assertIsNone(extract.valid_mnemonic("GoCompile\n")) + self.assertIsNone(extract.valid_enum("PASSED\n")) + self.assertIsNone(extract.valid_repo("@@foo+\n")) + + def test_valid_label_caps_length(self): + # The colon-separated target-name component is itself capped at 200 + # chars by LABEL_RE; spread the length over a long package path + # instead, to isolate MAX_LABEL_LEN (512) as the thing under test. + short_name = "b" * 190 + under_cap = "//" + ("a" * 300) + ":" + short_name # len 493 < 512 + over_cap = "//" + ("a" * 330) + ":" + short_name # len 523 > 512 + self.assertEqual(extract.valid_label(under_cap), under_cap) + self.assertIsNone(extract.valid_label(over_cap)) + + def test_valid_label_drops_tripwire_strings_individually(self): + self.assertIsNone(extract.valid_label("//pkg:x/tmp/y")) + self.assertIsNone(extract.valid_label("//pkg:x/etc/y")) + self.assertIsNone(extract.valid_label("//pkg:x/runner/y")) + + def test_valid_mnemonic_rejects_injected_text(self): + self.assertEqual(extract.valid_mnemonic("CppCompile"), "CppCompile") + self.assertIsNone(extract.valid_mnemonic("CppCompile; /home/runner/secret")) + + def test_valid_command_and_version(self): + self.assertEqual(extract.valid_command("test"), "test") + self.assertIsNone(extract.valid_command("test\nexport X=1")) + self.assertEqual(extract.valid_version("9.2.0"), "9.2.0") + self.assertIsNone(extract.valid_version("9.2.0; token=ghp_abc at http://evil")) + + def test_valid_uuid(self): + self.assertEqual( + extract.valid_uuid("11111111-2222-3333-4444-555555555555"), + "11111111-2222-3333-4444-555555555555", + ) + self.assertIsNone(extract.valid_uuid("ghp_SECRETSECRETSECRET0123456789abcd")) + self.assertIsNone(extract.valid_uuid("11111111-2222-3333-4444-555555555555\n")) + + def test_valid_lane(self): + self.assertEqual(extract.valid_lane("unit"), "unit") + self.assertEqual(extract.valid_lane("integration-packages"), "integration-packages") + self.assertIsNone(extract.valid_lane("unit Bearer abc.def")) + self.assertIsNone(extract.valid_lane("")) + + def test_as_int_handles_overflowing_float(self): + # A crafted numeric literal (e.g. 1e400) parses as float('inf'); + # int(inf) raises OverflowError, uncaught, before this fix. + self.assertEqual(extract.as_int(float("inf")), 0) + self.assertEqual(extract.as_int(float("-inf")), 0) + self.assertEqual(extract.as_int(float("nan")), 0) + + def test_runner_class(self): + self.assertEqual(extract.runner_class("remote cache hit"), "remote_cache_hit") + self.assertEqual(extract.runner_class("disk cache hit"), "disk_cache_hit") + self.assertEqual(extract.runner_class("remote"), "remote") + self.assertEqual(extract.runner_class("linux-sandbox"), "local") + self.assertEqual(extract.runner_class("worker"), "local") + self.assertEqual(extract.runner_class("something-unexpected"), "other") + + def test_clamp_pct_rejects_non_finite_and_out_of_range(self): + self.assertEqual(extract._clamp_pct(50.0), 50.0) + self.assertEqual(extract._clamp_pct(0.0), 0.0) + self.assertEqual(extract._clamp_pct(100.0), 100.0) + self.assertEqual(extract._clamp_pct(-5.0), 0.0) + self.assertEqual(extract._clamp_pct(150.0), 100.0) + self.assertEqual(extract._clamp_pct(float("inf")), 0.0) + self.assertEqual(extract._clamp_pct(float("-inf")), 0.0) + self.assertEqual(extract._clamp_pct(float("nan")), 0.0) + + +class RealFixtureTests(unittest.TestCase): + """testdata/ci_analytics/real/*: genuine Bazel 9.2.0 output.""" + + @staticmethod + def _fixture_critical_path_time(): + with open(os.path.join(REAL, "cold-bep.json"), encoding="utf-8") as f: + for line in f: + event = json.loads(line) + bm = event.get("buildMetrics") + if isinstance(bm, dict): + tm = bm.get("timingMetrics") + if isinstance(tm, dict) and "criticalPathTime" in tm: + return tm["criticalPathTime"] + return None + + def test_cold_invocation_has_a_failed_test_and_local_spawns(self): + rec = extract.build_invocation( + "test", + os.path.join(REAL, "cold-bep.json"), + os.path.join(REAL, "cold-exec.binpb"), + os.path.join(REAL, "cold-profile.json"), + _deadline(), + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["command"], "test") + self.assertEqual(rec["bazel_version"], "9.2.0") + self.assertEqual(rec["exec_log"], "ok") + self.assertEqual(rec["profile"], "ok") + self.assertGreater(rec["timing"]["wall_ms"], 0) + # criticalPathTime is a Duration string (e.g. "0.212852922s" in the + # currently-regenerated fixture), not criticalPathTimeInMs: as_int() + # on that string alone always silently produced 0 before this fix. + # Read the fixture's own value instead of hardcoding one, since + # regen.sh is nondeterministic run-to-run. + self.assertGreater(rec["timing"]["critical_path_ms"], 0) + self.assertEqual( + rec["timing"]["critical_path_ms"], + extract.as_duration_ms(self._fixture_critical_path_time()), + ) + + by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]} + self.assertIn("TestRunner", by_mnemonic) + self.assertIn("CppCompile", by_mnemonic) + # No RBE in this sandbox: every spawn in the cold run is local. + self.assertEqual(by_mnemonic["TestRunner"]["remote"], 0) + self.assertEqual(by_mnemonic["TestRunner"]["local"], by_mnemonic["TestRunner"]["spawns"]) + + labels = {t["label"]: t for t in rec["tests"]} + self.assertEqual(labels["//:flaky_test"]["overall"], "FAILED") + self.assertEqual(labels["//:pass_test"]["overall"], "PASSED") + self.assertEqual(labels["//:sharded_test"]["shards"], 3) + + # Per-test timing lives at testResult.executionInfo.timingBreakdown, + # not a top-level testResult.timingBreakdown (which doesn't exist): + # confirmed non-zero exec_ms for at least one real test result. + all_results = [r for t in rec["tests"] for r in t["results"]] + self.assertTrue(any(r["exec_ms"] > 0 for r in all_results)) + + # The *_pct fields are always present. They're each 0.0 on this + # fixture (a sandbox with no RBE endpoint: every percentage in the + # real "Critical Path: ...% of the time: [...]" breakdown is + # genuinely 0.00%, not just unparsed) -- the real assertion here is + # that build_timing() parsed the breakdown text at all (a test_* + # key below), not that any one percentage is non-zero. + for key in ( + "remote_pct", + "queue_pct", + "fetch_pct", + "process_pct", + "upload_pct", + "setup_pct", + "network_pct", + ): + self.assertIn(key, rec["critical_path"]) + self.assertNotEqual(rec["critical_path"]["test_label"], "") + + def test_postclean_invocation_is_mostly_disk_cache_hits(self): + rec = extract.build_invocation( + "test", + os.path.join(REAL, "postclean-bep.json"), + os.path.join(REAL, "postclean-exec.binpb"), + None, + _deadline(), + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "ok") + self.assertEqual(rec["profile"], "missing") + + by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]} + self.assertGreater(by_mnemonic["CppCompile"]["disk_cache_hit"], 0) + self.assertEqual(by_mnemonic["CppCompile"]["local"], 0) + # flaky_test's TestRunner spawn is never a disk-cache hit: Bazel + # does not cache a failed test, so it reruns (a cold row). + self.assertGreater(by_mnemonic["TestRunner"]["local"], 0) + cold_targets = {c["target"] for c in rec["cold"]} + self.assertIn("//:flaky_test", cold_targets) + + def test_repo_fetch_is_read_from_the_profile(self): + rec = extract.build_invocation( + "test", + os.path.join(REAL, "cold-bep.json"), + None, + os.path.join(REAL, "cold-profile.json"), + _deadline(), + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "missing") + self.assertEqual(rec["profile"], "ok") + # @@bazel_tools is always fetched (it's bootstrapped, not from a + # registry cache that might already be warm), so repo_fetch must + # be non-empty here: this is the must-fix regression test for + # trim_profile's single-line-document bug, where iter_repo_fetch() + # silently saw zero "Fetching repository" events no matter what was + # actually in the profile, and this assertion would have passed + # vacuously against an empty list. + self.assertTrue(rec["repo_fetch"]) + for row in rec["repo_fetch"]: + self.assertIn("repo", row) + self.assertIn("ms", row) + self.assertIsNotNone(extract.valid_repo(row["repo"])) + + def test_unreadable_profile_path_is_reported_accurately(self): + rec = extract.build_invocation( + "test", + os.path.join(REAL, "cold-bep.json"), + None, + os.path.join(REAL, "does-not-exist-profile.json"), + _deadline(), + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["profile"], "unreadable") + + # Event kinds the extractor itself ever reads out of a BEP event body + # (plus "id", which is on every event). Anything else showing up in a + # fixture is, by definition, something the extractor doesn't need -- + # and therefore something regen.sh's filter_bep() should have dropped. + _EVENT_KIND_ALLOW = { + "id", "started", "testResult", "testSummary", "buildMetrics", "finished", "buildToolLogs", + } + _STARTED_KEY_ALLOW = {"uuid", "command", "buildToolVersion", "startTimeMillis"} + # Keys that, wherever they appear in the structure, are specifically the + # ones known to carry operator/host/filesystem detail in real BEP output + # (test case URIs, output file URIs, exec hostnames, raw status/failure + # detail protos with command lines and paths baked in). + _FORBIDDEN_KEYS_ANY_DEPTH = {"uri", "testActionOutput", "hostname", "statusDetails", "failureDetail"} + + # client_env= (not CLIENT_ENVIRONMENT_VARIABLE, a legitimate Bazel + # skyfunction name that otherwise false-positives on "client_env"). + # Generic enough to catch leaks this fixture's own operator wouldn't + # produce (a different machine, a different token format, ...), not + # just the ones this particular sandbox happens to be able to leak. + _LEAK_PATTERN = extract.re.compile( + r"client_env=|\"host\"|\"user\"|BUILD_HOST|/Users/|/home/|ghp_|sk-|token|cherry|ubuntu", + extract.re.IGNORECASE, + ) + + def _walk_json_leak_check(self, obj, path, event_kinds_seen): + """Recursively asserts structural + string-content leak guards over + a decoded BEP event's JSON. event_kinds_seen collects top-level + event-kind keys seen (id/started/testResult/...) for the caller to + check against _EVENT_KIND_ALLOW.""" + if isinstance(obj, dict): + for key, value in obj.items(): + self.assertNotIn( + key, self._FORBIDDEN_KEYS_ANY_DEPTH, f"{path}.{key} should have been stripped by filter_bep()" + ) + self._walk_json_leak_check(value, f"{path}.{key}", event_kinds_seen) + elif isinstance(obj, list): + for i, value in enumerate(obj): + self._walk_json_leak_check(value, f"{path}[{i}]", event_kinds_seen) + elif isinstance(obj, str): + self.assertIsNone(extract.TRIPWIRE_RE.search(obj), f"{path} matches the document tripwire: {obj!r}") + self.assertIsNone(self._LEAK_PATTERN.search(obj), f"{path} matches the operator-leak pattern: {obj!r}") + + def test_fixtures_do_not_leak_the_operator_environment(self): + """The must-fix: a real fixture reviewer-readable on disk, not just + what the extractor happens to read, must carry nothing from the + operator's own environment, host, or session. + + Structural, not a raw-text denylist: a denylist tuned to one + operator misses things like --client_env=GH_TOKEN=ghp_..., a + different home directory, or an sk-... key. Here we walk the + *decoded* JSON of every BEP event, assert the event-kind and + started-key sets are each a subset of what the extractor actually + reads, assert none of the known leak-prone keys (uri, + testActionOutput, hostname, statusDetails, failureDetail) appear + at any nesting depth, and only then fall back to running the + denylist/tripwire over every decoded string value.""" + for name in ("cold-bep.json", "postclean-bep.json"): + path = os.path.join(REAL, name) + with open(path, encoding="utf-8") as f: + for lineno, line in enumerate(f, start=1): + line = line.strip() + if not line: + continue + event = json.loads(line) + self.assertIsInstance(event, dict, f"{name}:{lineno} is not an object") + kinds = set(event.keys()) + self.assertTrue( + kinds <= self._EVENT_KIND_ALLOW, + f"{name}:{lineno} has event kind(s) {kinds - self._EVENT_KIND_ALLOW} outside the allowlist", + ) + if "started" in event and isinstance(event["started"], dict): + started_keys = set(event["started"].keys()) + self.assertTrue( + started_keys <= self._STARTED_KEY_ALLOW, + f"{name}:{lineno} started has key(s) " + f"{started_keys - self._STARTED_KEY_ALLOW} outside the allowlist", + ) + self._walk_json_leak_check(event, f"{name}:{lineno}", kinds) + + # cold-profile.json is Bazel's own line-per-event trace format, not + # one-JSON-object-per-BEP-line; the leak checks on its string + # content still apply, but there's no BEP event-kind structure to + # validate. + with open(os.path.join(REAL, "cold-profile.json"), encoding="utf-8") as f: + text = f.read() + self.assertIsNone(extract.TRIPWIRE_RE.search(text), "cold-profile.json matches the document tripwire") + self.assertIsNone( + self._LEAK_PATTERN.search(text), "cold-profile.json matches the operator-leak pattern" + ) + + # The exec logs are zstd-compressed protobuf, not JSON -- decode as + # best-effort text (the same way `strings` would) and run the same + # leak pattern over whatever falls out, including non-UTF-8 noise. + for name in ("cold-exec.binpb", "postclean-exec.binpb"): + with open(os.path.join(REAL, name), "rb") as f: + data = f.read() + self.assertIsNone( + self._LEAK_PATTERN.search(data.decode("utf-8", "replace")), + f"{name} matches the operator-leak pattern", + ) + + +class SyntheticRemoteFixtureTests(unittest.TestCase): + """A synthetic exec log + BEP for the `remote` and `remote cache hit` + runner classes, which this sandbox cannot produce against a real RBE + endpoint (design doc section 3.5).""" + + def setUp(self): + self.tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(self.tmpdir.cleanup) + self.uuid = "00000000-0000-0000-0000-000000000001" + self.bep_path = os.path.join(self.tmpdir.name, "bep.json") + self.exec_log_path = os.path.join(self.tmpdir.name, "exec.log") + with open(self.bep_path, "w", encoding="utf-8") as f: + f.write(synthetic_bep(self.uuid)) + with open(self.exec_log_path, "wb") as f: + f.write(synthetic_exec_log(self.uuid)) + + def test_remote_and_remote_cache_hit_spawns_are_classified(self): + rec = extract.build_invocation("build", self.bep_path, self.exec_log_path, None, _deadline()) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "ok") + self.assertEqual(rec["invocation_id"], self.uuid) + # "0.900s" decodes via as_duration_ms to 900ms exactly. + self.assertEqual(rec["timing"]["critical_path_ms"], 900) + + by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]} + cpp = by_mnemonic["CppCompile"] + self.assertEqual(cpp["remote"], 1) + self.assertEqual(cpp["remote_cache_hit"], 1) + self.assertEqual(cpp["remote_queue_ms"], 250) + self.assertEqual(cpp["remote_queue_max_ms"], 250) + self.assertEqual(cpp["remote_exec_ms"], 500) + + cold_targets = {c["target"]: c for c in rec["cold"]} + self.assertIn("//:synth_remote", cold_targets) + self.assertNotIn("//:synth_cached", cold_targets) # a hit, not cold + + +SECRET_MARKERS = ( + "ghp_SECRETTOKEN1234567890", + "evil.example.com", + "attacker.example", + "AWS_SECRET_ACCESS_KEY", + "grpcs://rbe-west-internal.ops.gascity.com", + "/etc/secrets/rbe-client.key", + "DEPLOY_TOKEN", + "s3cr3t-deploy-token-0001", + "/root/.ssh/id_rsa", + "10.0.0.5", + "ghp_abc123", + "BEGIN PRIVATE KEY", + "/home/runner/secret", + "leak.example.com", + "hunter2", +) + + +class PoisonedBEPTests(unittest.TestCase): + """testdata/ci_analytics/poisoned-bep.json: every field the extractor + reads carries a secret or a leaky value, including two hostile labels + (one tripwire-triggering, one with injected text in its status/ + runnerCount fields). The summary must contain none of them, whether + through whitelist validation or the tripwire -- and a hostile label + must only drop that one row, not the whole document.""" + + def test_poisoned_bep_leaks_nothing(self): + rec = extract.build_invocation( + "test", os.path.join(TESTDATA, "poisoned-bep.json"), None, None, _deadline() + ) + self.assertIsNotNone(rec) + text = json.dumps(rec, sort_keys=True) + for marker in SECRET_MARKERS: + self.assertNotIn(marker, text, f"leaked {marker!r} into the summary") + self.assertIsNone(extract.TRIPWIRE_RE.search(text)) + # The poisoned fields were dropped, not merely escaped. + self.assertEqual(rec["command"], "unknown") + self.assertEqual(rec["bazel_version"], "unknown") + self.assertEqual(rec["exit_name"], "UNKNOWN") + # The hostile /etc/-labeled testSummary was dropped entirely, but + # the rest of the document (e.g. //:pass_test) is still present: + # one poisoned label does not kill the whole document. + labels = {t["label"] for t in rec["tests"]} + self.assertNotIn("//pkg:x/etc/evil_test", labels) + self.assertIn("//:pass_test", labels) + + def test_full_summary_build_rejects_nothing_silently_wrong(self): + # The end-to-end path (build_summary -> the tripwire over the + # serialized output) also sees no leak and still writes a summary + # (poisoning a field must not suppress the whole invocation). + summary = extract.build_summary( + "unit", + "local", + "123", + "0", + [("test", os.path.join(TESTDATA, "poisoned-bep.json"), None, None)], + _deadline(), + ) + self.assertIsNotNone(summary) + text = json.dumps(summary, sort_keys=True) + for marker in SECRET_MARKERS: + self.assertNotIn(marker, text, f"leaked {marker!r} into the summary") + + def test_poisoned_exec_log_leaks_nothing(self): + """Secrets in a spawn's args/env, next to a hostile mnemonic and a + hostile target, built with the test-side varint encoder: args/env + (fields 1/2) are never decoded at all, and the hostile + mnemonic/target are dropped by valid_mnemonic()/valid_label().""" + uuid = "00000000-0000-0000-0000-00000000bad1" + exec_log = _invocation_entry(uuid) + _entry_with_spawn( + _spawn( + "//pkg:x/etc/evil", # hostile target: tripwire substring + "Evil; /home/runner/secret", # hostile mnemonic: invalid charset + "local", + False, + _spawn_metrics(total_ms=10, exec_ms=10), + # A label-shaped secret: if a future regression ever + # decoded args (field 1) into the label/target used + # anywhere downstream (e.g. mistaking an argv[] entry for + # the target), this would surface as a leaked + # //:hunter2_secret row rather than silently passing. + args=("--token=hunter2", "//:hunter2_secret", "AKIAIOSFODNN7EXAMPLE"), + env=(("DEPLOY_TOKEN", "s3cr3t-deploy-token-0001"),), + ) + ) + with tempfile.TemporaryDirectory() as tmp: + bep_path = os.path.join(tmp, "bep.json") + exec_path = os.path.join(tmp, "exec.log") + with open(bep_path, "w", encoding="utf-8") as f: + f.write(synthetic_bep(uuid)) + with open(exec_path, "wb") as f: + f.write(exec_log) + rec = extract.build_invocation("test", bep_path, exec_path, None, _deadline()) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "ok") + self.assertEqual(rec["mnemonics"], []) # the hostile mnemonic was dropped + self.assertEqual(rec["cold"], []) # and so is the only spawn carrying it + text = json.dumps(rec, sort_keys=True) + for marker in ( + "hunter2", + "hunter2_secret", + "AKIAIOSFODNN7EXAMPLE", + "DEPLOY_TOKEN", + "s3cr3t-deploy-token-0001", + ): + self.assertNotIn(marker, text) + + def test_poisoned_profile_leaks_nothing(self): + # One event per line -- {"traceEvents":[\n{...},\n{...}\n]} -- the + # real shape Bazel's own --profile output is in, and the shape + # iter_repo_fetch() actually reads line by line: a single + # json.dump() here (one line for the whole document) would make + # every "Fetching repository" entry live on the one line that also + # contains the "traceEvents" key, whose top-level dict has no "cat" + # of its own, so it never reaches ev.get("cat") == "Fetching + # repository" at all. repo_fetch would always come back empty and + # this test would pass vacuously, without ever exercising + # valid_repo() against the hostile name. + hostile = { + "cat": "Fetching repository", + "name": "@@evil+; token=hunter2 /etc/passwd", + "ts": 0, + "dur": 1000, + } + valid = {"cat": "Fetching repository", "name": "@@rules_cc+", "ts": 2000, "dur": 500} + with tempfile.TemporaryDirectory() as tmp: + bep_path = os.path.join(tmp, "bep.json") + profile_path = os.path.join(tmp, "profile.json") + with open(bep_path, "w", encoding="utf-8") as f: + f.write(synthetic_bep("00000000-0000-0000-0000-00000000bad2")) + with open(profile_path, "w", encoding="utf-8") as f: + f.write('{"traceEvents":[\n') + f.write(",\n".join(json.dumps(e) for e in (hostile, valid))) + f.write("\n]}\n") + rec = extract.build_invocation("test", bep_path, None, profile_path, _deadline()) + self.assertIsNotNone(rec) + self.assertEqual(rec["profile"], "ok") + repos = {r["repo"] for r in rec["repo_fetch"]} + self.assertNotIn("@@evil+; token=hunter2 /etc/passwd", repos) # the hostile repo name was dropped + self.assertIn("@@rules_cc+", repos) # its valid sibling survives + text = json.dumps(rec, sort_keys=True) + self.assertNotIn("hunter2", text) + + def test_one_tripwire_label_drops_only_that_row(self): + lines = [ + json.dumps( + { + "started": { + "uuid": "11111111-2222-3333-4444-555555555555", + "command": "test", + "buildToolVersion": "9.2.0", + } + } + ), + json.dumps( + { + "id": {"testSummary": {"label": "//internal/runner/tmp:x/etc/y_test"}}, + "testSummary": {"overallStatus": "PASSED"}, + } + ), + json.dumps( + { + "id": {"testSummary": {"label": "//:fine_test"}}, + "testSummary": {"overallStatus": "PASSED"}, + } + ), + ] + with tempfile.TemporaryDirectory() as tmp: + bep_path = os.path.join(tmp, "t.bep.json") + out_path = os.path.join(tmp, "t.out.json") + with open(bep_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + rc = extract.main( + [ + "--lane", "unit", "--mode", "local", "--check-run-id", "1", "--pr-hint", "0", + "--invocation", "test", bep_path, "--out", out_path, + ] + ) + self.assertEqual(rc, 0) + with open(out_path, encoding="utf-8") as f: + summary = json.load(f) + labels = {t["label"] for t in summary["invocations"][0]["tests"]} + self.assertNotIn("//internal/runner/tmp:x/etc/y_test", labels) + self.assertIn("//:fine_test", labels) + + +class CLITests(unittest.TestCase): + def test_end_to_end_writes_a_schema_v1_file(self): + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "nested", "summary.json") + rc = extract.main( + [ + "--lane", + "unit", + "--mode", + "local", + "--check-run-id", + "42", + "--pr-hint", + "7", + "--invocation", + "test", + os.path.join(REAL, "cold-bep.json"), + os.path.join(REAL, "cold-exec.binpb"), + os.path.join(REAL, "cold-profile.json"), + "--out", + out, + ] + ) + self.assertEqual(rc, 0) + with open(out, encoding="utf-8") as f: + summary = json.load(f) + self.assertEqual(summary["format"], "ci-analytics-summary") + self.assertEqual(summary["version"], 1) + self.assertEqual(summary["lane"], "unit") + self.assertEqual(summary["check_run_id"], 42) + self.assertEqual(len(summary["invocations"]), 1) + + def test_no_bep_anywhere_exits_3_and_writes_nothing(self): + with tempfile.TemporaryDirectory() as tmp: + missing = os.path.join(tmp, "nope.json") + out = os.path.join(tmp, "summary.json") + rc = extract.main( + [ + "--lane", + "unit", + "--mode", + "local", + "--check-run-id", + "1", + "--pr-hint", + "0", + "--invocation", + "test", + missing, + "--out", + out, + ] + ) + self.assertEqual(rc, 3) + self.assertFalse(os.path.exists(out)) + + def test_too_many_invocations_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "summary.json") + argv = ["--lane", "unit", "--mode", "local", "--check-run-id", "1", "--pr-hint", "0", "--out", out] + for i in range(extract.MAX_INVOCATIONS + 1): + argv += ["--invocation", f"step{i}", os.path.join(REAL, "cold-bep.json")] + with self.assertRaises(SystemExit): + extract.parse_args(argv) + + def test_hostile_lane_mode_step_and_uuid_are_rejected_or_dropped(self): + with tempfile.TemporaryDirectory() as tmp: + bep_path = os.path.join(tmp, "u.bep.json") + out = os.path.join(tmp, "u.out.json") + hostile_uuid = "ghp_SECRETSECRETSECRET0123456789abcd" + with open(bep_path, "w", encoding="utf-8") as f: + f.write( + json.dumps( + {"started": {"uuid": hostile_uuid, "command": "test", "buildToolVersion": "9.2.0"}} + ) + + "\n" + ) + # --lane, --mode and STEP with injected auth-header-shaped text: + # rejected at argument-parsing time, before any file is read. + with self.assertRaises(SystemExit): + extract.main( + [ + "--lane", "unit Bearer abc.def", + "--mode", "remote token=hunter2", + "--check-run-id", "1", "--pr-hint", "0", + "--invocation", "step with spaces AKIAIOSFODNN7EXAMPLE", bep_path, + "--out", out, + ] + ) + # A valid --lane/--mode/STEP, but a hostile started.uuid: the + # CLI still succeeds, but the hostile uuid never reaches the + # output (dropped to "", not validated-and-passed-through). + rc = extract.main( + [ + "--lane", "unit", "--mode", "remote", "--check-run-id", "1", "--pr-hint", "0", + "--invocation", "test", bep_path, "--out", out, + ] + ) + self.assertEqual(rc, 0) + with open(out, encoding="utf-8") as f: + text = f.read() + self.assertNotIn(hostile_uuid, text) + + def test_mode_enum_is_enforced(self): + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "summary.json") + with self.assertRaises(SystemExit): + extract.parse_args( + [ + "--lane", "unit", "--mode", "not-a-real-mode", "--check-run-id", "1", + "--pr-hint", "0", "--invocation", "test", os.path.join(REAL, "cold-bep.json"), + "--out", out, + ] + ) + + +class ElapsedTimeFieldTests(unittest.TestCase): + """buildToolLogs' base64-encoded "elapsed time" / "critical path" logs + are read; make sure a benign one decodes correctly (the malicious case + is covered by PoisonedBEPTests).""" + + def test_elapsed_and_critical_path_decode(self): + text = ( + "Critical Path: 2.50s, Remote (10.0% of the time): [queue: 1.50%, process: 2.25%]\n" + " 2.50s action 'Testing //:pass_test (shard 0 of 1)'\n" + ) + events = [ + {"id": {"started": {}}, "started": {"uuid": "u", "command": "test", "buildToolVersion": "9.2.0"}}, + { + "id": {"buildToolLogs": {}}, + "buildToolLogs": { + "log": [ + {"name": "elapsed time", "contents": base64.b64encode(b"2.5").decode()}, + {"name": "critical path", "contents": base64.b64encode(text.encode()).decode()}, + ] + }, + }, + ] + started, finished, timing, critical_path, actions, graph, net = extract.build_timing(events) + self.assertEqual(timing["elapsed_ms"], 2500) + self.assertEqual(critical_path["remote_pct"], 10.0) + self.assertEqual(critical_path["queue_pct"], 1.50) + self.assertEqual(critical_path["process_pct"], 2.25) + self.assertEqual(critical_path["test_label"], "//:pass_test") + self.assertEqual(critical_path["test_shard"], 0) + self.assertEqual(critical_path["test_ms"], 2500) + + def test_critical_path_test_line_matches_unsharded_tests_too(self): + text = ( + "Critical Path: 1.00s, Remote (0.0% of the time): []\n" + " 1.00s action 'Testing //:unsharded_test'\n" + ) + events = [ + {"id": {"started": {}}, "started": {"uuid": "u", "command": "test", "buildToolVersion": "9.2.0"}}, + { + "id": {"buildToolLogs": {}}, + "buildToolLogs": { + "log": [{"name": "critical path", "contents": base64.b64encode(text.encode()).decode()}] + }, + }, + ] + _, _, _, critical_path, _, _, _ = extract.build_timing(events) + self.assertEqual(critical_path["test_label"], "//:unsharded_test") + self.assertEqual(critical_path["test_shard"], 0) + + def test_critical_path_pct_keys_always_present_even_with_no_match(self): + events = [ + {"id": {"started": {}}, "started": {"uuid": "u", "command": "test", "buildToolVersion": "9.2.0"}}, + ] + _, _, _, critical_path, _, _, _ = extract.build_timing(events) + for key in ( + "remote_pct", + "queue_pct", + "fetch_pct", + "process_pct", + "upload_pct", + "setup_pct", + "network_pct", + ): + self.assertEqual(critical_path[key], 0.0) + # Always present (the S4 contract), even with no critical-path text + # at all: "" / 0 defaults, not an absent key. + self.assertEqual(critical_path["test_label"], "") + self.assertEqual(critical_path["test_shard"], 0) + self.assertEqual(critical_path["test_ms"], 0) + + +class RobustnessTests(unittest.TestCase): + """Every crash a reviewer's adversarial script reproduced against this + extractor must degrade (a status string, or a smaller/partial result), + never raise out of build_invocation().""" + + def _bep(self, uuid="11111111-2222-3333-4444-555555555555", extra_started=None, lines_after=()): + st = {"uuid": uuid, "command": "test", "buildToolVersion": "9.2.0", "startTimeMillis": "1"} + if extra_started: + st.update(extra_started) + lines = [json.dumps({"id": {"started": {}}, "started": st})] + list(lines_after) + return "\n".join(lines) + "\n" + + def _run(self, bep_text, exec_bytes, deadline=None): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(bep_text) + ep = None + if exec_bytes is not None: + ep = os.path.join(tmp, "x.exec") + with open(ep, "wb") as f: + f.write(exec_bytes) + return extract.build_invocation("test", bp, ep, None, deadline if deadline is not None else _deadline()) + + def test_truncated_varint_tail_degrades_to_corrupt(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + rec = self._run(self._bep(uuid), inv_entry + b"\x80") + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "corrupt") + + def test_wiretype_3_in_entry_degrades_to_corrupt(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + rec = self._run(self._bep(uuid), inv_entry + _length_prefixed(b"\x3b")) # field 7, wire 3 + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "corrupt") + + def test_wiretype_6_in_entry_degrades_to_corrupt(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + rec = self._run(self._bep(uuid), inv_entry + _length_prefixed(b"\x3e")) # field 7, wire 6 + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "corrupt") + + def test_spawn_target_as_varint_is_skipped_not_crashed(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + # Spawn.target_label (field 7) re-encoded as a varint instead of a + # length-delimited string. + entry = _length_prefixed(b"\x3a" + _length_prefixed(b"\x38\x01")) + rec = self._run(self._bep(uuid), inv_entry + entry) + self.assertIsNotNone(rec) + self.assertIn(rec["exec_log"], ("ok", "corrupt")) + + def test_metrics_input_bytes_as_bytes_is_skipped_not_crashed(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + metrics = b"\x42" + _length_prefixed(b"\x5a" + _length_prefixed(b"xx")) + entry = _length_prefixed(b"\x3a" + _length_prefixed(metrics)) + rec = self._run(self._bep(uuid), inv_entry + entry) + self.assertIsNotNone(rec) + self.assertIn(rec["exec_log"], ("ok", "corrupt")) + + def test_invocation_submessage_as_varint_degrades(self): + rec = self._run(self._bep(), _length_prefixed(b"\x10\x05")) + self.assertIsNotNone(rec) + self.assertIn(rec["exec_log"], ("ok", "mismatch", "corrupt")) + + def test_huge_length_prefix_does_not_crash(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + rec = self._run(self._bep(uuid), inv_entry + _varint(2**62) + b"abc") + self.assertIsNotNone(rec) + + def test_deadline_is_checked_inside_a_single_oversized_submessage(self): + """spawns()'s own deadline check fires once per ExecLogEntry -- but + a single entry can itself be made arbitrarily large (a reviewer's + adversarial script built one 40 MB Spawn out of millions of tiny + 2-byte fields and measured 15.8s against a 5s deadline, since + nothing inside fields() ever looked at the clock). Build one + Spawn's SpawnMetrics submessage (field 18) out of many tiny unknown + fields -- small enough to construct quickly in a test, but still + many times DEADLINE_CHECK_EVERY -- and confirm an already-elapsed + deadline is caught well before the whole submessage is decoded.""" + uuid = "11111111-2222-3333-4444-555555555555" + # field 1 (total_ms, wire 0): 2 bytes/field (tag + a 1-byte value). + huge_metrics = b"\x08\x00" * (extract.DEADLINE_CHECK_EVERY * 5) + spawn_body = _spawn("//:huge", "CppCompile", "local", False, huge_metrics) + exec_log = _invocation_entry(uuid) + _entry_with_spawn(spawn_body) + start = time.monotonic() + rec = self._run(self._bep(uuid), exec_log, deadline=time.monotonic() - 1) + elapsed = time.monotonic() - start + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "timeout") + self.assertLess(elapsed, 5.0) + + def test_bep_line_is_a_list(self): + rec = self._run(self._bep(lines_after=["[1,2,3]", '{"finished":{}}']), None) + self.assertIsNotNone(rec) + + def test_bep_line_is_a_bare_string(self): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write('"started here"\n{"x":1}\n') + rec = extract.build_invocation("test", bp, None, None, _deadline()) + self.assertIsNone(rec) # no valid started event anywhere: dropped, not crashed + + def test_bep_inf_number_in_started(self): + text = self._bep(extra_started={"startTimeMillis": 1e999}).replace("Infinity", "1e999") + rec = self._run(text, None) + self.assertIsNotNone(rec) + self.assertEqual(rec["started_ms"], 0) # as_int() degraded the overflow to the default + + def test_bep_started_not_a_dict(self): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write('{"started": 5}\n') + rec = extract.build_invocation("test", bp, None, None, _deadline()) + self.assertIsNone(rec) # started never became valid: dropped, not crashed + + def test_critpath_bad_float_does_not_crash(self): + cp = "Critical Path: 1.2.3s, Remote (0.0% of the time): []\n" + rec = self._run( + self._bep( + lines_after=[ + json.dumps( + { + "buildToolLogs": { + "log": [ + { + "name": "critical path", + "contents": base64.b64encode(cp.encode()).decode(), + } + ] + } + } + ) + ] + ), + None, + ) + self.assertIsNotNone(rec) + + def test_testresult_not_a_dict(self): + rec = self._run( + self._bep( + lines_after=[ + json.dumps({"id": {"testResult": {"label": "//:a"}}, "testResult": "PASSED"}) + ] + ), + None, + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["tests"], []) # the malformed event was skipped + + def test_runnercount_count_overflow_does_not_crash(self): + rec = self._run( + self._bep( + lines_after=[ + '{"buildMetrics":{"actionSummary":{"runnerCount":[{"name":"remote","count":1e999}]}}}' + ] + ), + None, + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["actions"]["remote"], 0) # the overflow degraded to the default + + def _profile_with_event(self, event_line): + return '{"traceEvents":[\n' + event_line + "\n]}\n" + + def test_repo_fetch_infinite_ts_does_not_crash(self): + # 1e999 parses as float('inf') via json's default Infinity literal + # (iter_repo_fetch does not reject non-finite constants the way + # read_bep() does -- it must instead catch them with isfinite()). + line = json.dumps({"cat": "Fetching repository", "name": "@@evil", "ts": 1e999, "dur": 1}).replace( + "Infinity", "1e999" + ) + with tempfile.TemporaryDirectory() as tmp: + pp = os.path.join(tmp, "p.json") + with open(pp, "w", encoding="utf-8") as f: + f.write(self._profile_with_event(line)) + rows, status = extract.iter_repo_fetch(pp, _deadline()) + self.assertEqual(rows, []) + self.assertEqual(status, "corrupt") + + def test_repo_fetch_nan_ts_does_not_crash(self): + line = json.dumps({"cat": "Fetching repository", "name": "@@evil", "ts": float("nan"), "dur": 1}) + with tempfile.TemporaryDirectory() as tmp: + pp = os.path.join(tmp, "p.json") + with open(pp, "w", encoding="utf-8") as f: + f.write(self._profile_with_event(line)) + rows, status = extract.iter_repo_fetch(pp, _deadline()) + self.assertEqual(rows, []) + self.assertEqual(status, "corrupt") + + def test_repo_fetch_overflowing_sum_does_not_crash(self): + # ts and dur are each individually finite (1e308 is a valid + # float), but their sum overflows a double to inf: isfinite(start) + # alone would miss this, only isfinite(end) catches it. + line = json.dumps({"cat": "Fetching repository", "name": "@@evil", "ts": 1e308, "dur": 1e308}) + with tempfile.TemporaryDirectory() as tmp: + pp = os.path.join(tmp, "p.json") + with open(pp, "w", encoding="utf-8") as f: + f.write(self._profile_with_event(line)) + rows, status = extract.iter_repo_fetch(pp, _deadline()) + self.assertEqual(rows, []) + self.assertEqual(status, "corrupt") + + def test_repo_fetch_corrupt_entry_does_not_hide_valid_siblings(self): + bad = json.dumps({"cat": "Fetching repository", "name": "@@evil", "ts": float("nan"), "dur": 1}) + good = json.dumps({"cat": "Fetching repository", "name": "@@rules_cc+", "ts": 0, "dur": 500}) + with tempfile.TemporaryDirectory() as tmp: + pp = os.path.join(tmp, "p.json") + with open(pp, "w", encoding="utf-8") as f: + f.write('{"traceEvents":[\n' + ",\n".join((bad, good)) + "\n]}\n") + rows, status = extract.iter_repo_fetch(pp, _deadline()) + self.assertEqual(status, "corrupt") + self.assertEqual([r["repo"] for r in rows], ["@@rules_cc+"]) + + +class BoundsTests(unittest.TestCase): + def test_exec_log_parse_timeout_is_reported(self): + uuid = "11111111-2222-3333-4444-555555555555" + exec_log = _invocation_entry(uuid) + # DEADLINE_CHECK_EVERY (2000) + a margin of spawn entries, so the + # deadline check inside spawns() fires at least once. The BEP + # itself is a single line, so read_bep()'s own (much coarser) + # check never fires even with an already-elapsed deadline: the + # timeout this test exercises is specifically the exec-log pass'. + for i in range(extract.DEADLINE_CHECK_EVERY + 50): + exec_log += _entry_with_spawn( + _spawn(f"//:t{i}", "CppCompile", "local", False, _spawn_metrics(total_ms=1)) + ) + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + ep = os.path.join(tmp, "x.exec") + with open(bp, "w", encoding="utf-8") as f: + f.write( + json.dumps( + { + "id": {"started": {}}, + "started": { + "uuid": uuid, + "command": "test", + "buildToolVersion": "9.2.0", + "startTimeMillis": "1", + }, + } + ) + + "\n" + ) + with open(ep, "wb") as f: + f.write(exec_log) + # An already-elapsed deadline: once the spawn loop's entry + # count crosses DEADLINE_CHECK_EVERY, it must raise + # ParseTimeout, which build_invocation turns into an + # "exec_log": "timeout" status rather than propagating. + rec = extract.build_invocation("test", bp, ep, None, time.monotonic() - 1) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "timeout") + + @unittest.skipUnless(shutil.which("zstd"), "zstd not on PATH") + def test_zstd_round_trip(self): + uuid = "11111111-2222-3333-4444-555555555555" + raw = synthetic_exec_log(uuid) + with tempfile.TemporaryDirectory() as tmp: + compressed = os.path.join(tmp, "exec.log.zst") + proc = subprocess.run( + ["zstd", "-q", "-o", compressed], + input=raw, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + del proc + data, status = extract.load_exec_log(compressed, _deadline()) + self.assertEqual(status, "ok") + self.assertEqual(data, raw) + + @unittest.skipUnless(shutil.which("zstd"), "zstd not on PATH") + def test_zstd_decompression_bomb_is_bounded(self): + # Patch MAX_EXEC_LOG_BYTES down instead of actually decompressing + # past the real 512 MiB cap: this test only needs to prove + # load_exec_log() stops reading once the decompressed stream + # crosses the cap, not that the cap is specifically 512 MiB, and a + # 600 MiB bomb was slow to build and (briefly) held 512 MiB+ in + # this process regardless of the outcome. + old_cap = extract.MAX_EXEC_LOG_BYTES + extract.MAX_EXEC_LOG_BYTES = 8 * 1024 * 1024 # 8 MiB + try: + with tempfile.TemporaryDirectory() as tmp: + bomb = os.path.join(tmp, "bomb.zst") + # 64 MiB of zeros, well past the patched 8 MiB cap, + # compresses to a tiny file almost instantly. + dd = subprocess.Popen( + ["dd", "if=/dev/zero", "bs=1M", "count=64"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + with open(bomb, "wb") as out: + zstd = subprocess.Popen(["zstd", "-q", "-1"], stdin=dd.stdout, stdout=out) + dd.stdout.close() + zstd.communicate(timeout=60) + dd.wait(timeout=60) + start = time.monotonic() + data, status = extract.load_exec_log(bomb, time.monotonic() + 30) + elapsed = time.monotonic() - start + finally: + extract.MAX_EXEC_LOG_BYTES = old_cap + self.assertIsNone(data) + self.assertEqual(status, "too_large") + self.assertLess(elapsed, 30.0) + + +class ContractTests(unittest.TestCase): + """The S4 contract: a stable, always-complete key set and status + vocabulary, so the collector never has to guess at a missing field.""" + + def test_max_test_results_is_a_global_total_not_per_label(self): + lines = [ + json.dumps( + { + "id": {"started": {}}, + "started": { + "uuid": "u", + "command": "test", + "buildToolVersion": "9.2.0", + "startTimeMillis": "1", + }, + } + ) + ] + # Two labels, each with more rows than half of a small total cap: + # if the cap were still enforced per-label, both labels would fit + # under it individually; enforced globally, the combined total must + # be clipped. + old_cap = extract.MAX_TEST_RESULTS + extract.MAX_TEST_RESULTS = 10 + try: + for label_i in range(2): + for run in range(8): + lines.append( + json.dumps( + { + "id": { + "testResult": { + "label": f"//:label{label_i}", + "run": run, + "shard": 0, + "attempt": 1, + } + }, + "testResult": {"status": "PASSED", "testAttemptDurationMillis": "1"}, + } + ) + ) + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + rec = extract.build_invocation("test", bp, None, None, _deadline()) + finally: + extract.MAX_TEST_RESULTS = old_cap + self.assertIsNotNone(rec) + total = sum(len(t["results"]) for t in rec["tests"]) + self.assertLessEqual(total, 10) + self.assertGreater(rec["tests_truncated"], 0) + + def test_exit_code_is_minus_one_without_a_finished_event(self): + rec = self._run_minimal_bep() + self.assertEqual(rec["exit_code"], -1) + self.assertEqual(rec["exit_name"], "UNKNOWN") + + def test_profile_missing_vs_unreadable_vs_ok(self): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(self._minimal_bep_text()) + rec_missing = extract.build_invocation("test", bp, None, None, _deadline()) + rec_unreadable = extract.build_invocation( + "test", bp, None, os.path.join(tmp, "nope.json"), _deadline() + ) + profile_path = os.path.join(tmp, "profile.json") + with open(profile_path, "w", encoding="utf-8") as f: + json.dump({"traceEvents": []}, f) + rec_ok = extract.build_invocation("test", bp, None, profile_path, _deadline()) + self.assertEqual(rec_missing["profile"], "missing") + self.assertEqual(rec_unreadable["profile"], "unreadable") + self.assertEqual(rec_ok["profile"], "ok") + + def test_exec_log_missing_vs_unreadable(self): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(self._minimal_bep_text()) + rec_missing = extract.build_invocation("test", bp, None, None, _deadline()) + rec_unreadable = extract.build_invocation( + "test", bp, os.path.join(tmp, "nope.exec"), None, _deadline() + ) + self.assertEqual(rec_missing["exec_log"], "missing") + self.assertEqual(rec_unreadable["exec_log"], "unreadable") + + @staticmethod + def _minimal_bep_text(): + return ( + json.dumps( + { + "id": {"started": {}}, + "started": { + "uuid": "11111111-2222-3333-4444-555555555555", + "command": "test", + "buildToolVersion": "9.2.0", + "startTimeMillis": "1", + }, + } + ) + + "\n" + ) + + def _run_minimal_bep(self): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(self._minimal_bep_text()) + return extract.build_invocation("test", bp, None, None, _deadline()) + + +class BEPReadContractTests(unittest.TestCase): + """The malformed-middle-line vs. truncated-last-line BEP contract + (shared with gascity's bep.go / beads' extractor), plus the ported + beads invocation-id mismatch fixture.""" + + def _started_line(self, uuid): + return json.dumps( + { + "id": {"started": {}}, + "started": { + "uuid": uuid, + "command": "test", + "buildToolVersion": "9.2.0", + "startTimeMillis": "1", + }, + } + ) + + def test_malformed_middle_line_drops_the_invocation(self): + uuid = "11111111-2222-3333-4444-555555555555" + text = self._started_line(uuid) + "\n{not json\n" + '{"finished":{}}\n' + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(text) + rec = extract.build_invocation("test", bp, None, None, _deadline()) + self.assertIsNone(rec) + + def test_truncated_last_line_sets_bep_truncated(self): + uuid = "11111111-2222-3333-4444-555555555555" + text = self._started_line(uuid) + "\n" + '{"finished": {"exitCode": {"cod' # cut off mid-line + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(text) + rec = extract.build_invocation("test", bp, None, None, _deadline()) + self.assertIsNotNone(rec) + self.assertTrue(rec["bep_truncated"]) + + def test_beads_mismatch_fixture_reports_mismatch(self): + """Ported from beads' tools/bazel/testdata/ci_analytics/ + mismatch.bep.jsonl: a BEP whose started.uuid does not match the + exec log's embedded invocation id.""" + bep_path = os.path.join(TESTDATA, "mismatch.bep.jsonl") + with open(bep_path, encoding="utf-8") as f: + text = f.read() + self.assertIn("99999999-0000-1111-2222-333333333333", text) + exec_log = synthetic_exec_log("00000000-aaaa-bbbb-cccc-000000000000") + with tempfile.TemporaryDirectory() as tmp: + ep = os.path.join(tmp, "mismatch.exec") + with open(ep, "wb") as f: + f.write(exec_log) + rec = extract.build_invocation("test", bep_path, ep, None, _deadline()) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "mismatch") + self.assertEqual(rec["mnemonics"], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/bazel/testdata/ci_analytics/mismatch.bep.jsonl b/tools/bazel/testdata/ci_analytics/mismatch.bep.jsonl new file mode 100644 index 000000000..37680e98d --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/mismatch.bep.jsonl @@ -0,0 +1,3 @@ +{"started": {"uuid": "99999999-0000-1111-2222-333333333333", "command": "test", "buildToolVersion": "9.2.0", "startTimeMillis": "2000"}} +{"buildMetrics": {"timingMetrics": {"wallTimeInMs": "15"}, "actionSummary": {"actionsCreated": "1", "actionsExecuted": "1"}}} +{"finished": {"exitCode": {"code": 0, "name": "SUCCESS"}, "finishTimeMillis": "2015"}} diff --git a/tools/bazel/testdata/ci_analytics/poisoned-bep.json b/tools/bazel/testdata/ci_analytics/poisoned-bep.json new file mode 100644 index 000000000..258a06e7e --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/poisoned-bep.json @@ -0,0 +1,8 @@ +{"id": {"started": {}}, "started": {"uuid": "poison-0000-0000-0000-000000000000", "startTimeMillis": "1700000000000", "buildToolVersion": "9.2.0; TOKEN=ghp_SECRETTOKEN1234567890 at http://evil.example.com/leak", "command": "test\nexport AWS_SECRET_ACCESS_KEY=abcd1234 && curl https://attacker.example/x"}} +{"id": {"unstructuredCommandLine": {}}, "unstructuredCommandLine": {"args": ["test", "--remote_executor=grpcs://rbe-west-internal.ops.gascity.com:443", "--tls_client_key=/etc/secrets/rbe-client.key", "--test_env=DEPLOY_TOKEN=s3cr3t-deploy-token-0001"]}} +{"id": {"testResult": {"label": "//:pass_test", "run": 1, "shard": 0, "attempt": 1}}, "testResult": {"status": "PASSED; /root/.ssh/id_rsa leaked, see 10.0.0.5", "executionInfo": {"strategy": "remote cache hit; token=ghp_abc123"}, "cachedLocally": false, "testAttemptDurationMillis": "123"}} +{"id": {"testSummary": {"label": "//:pass_test"}}, "testSummary": {"overallStatus": "PASSED", "shardCount": 1, "attemptCount": 1, "totalNumCached": 0}} +{"id": {"buildMetrics": {}}, "buildMetrics": {"timingMetrics": {"cpuTimeInMs": "100", "wallTimeInMs": "200"}, "actionSummary": {"actionsCreated": "5", "actionsExecuted": "5", "runnerCount": [{"name": "remote; -----BEGIN PRIVATE KEY----- leak", "count": "5", "execKind": "Remote"}], "actionData": [{"mnemonic": "CppCompile; /home/runner/secret", "actionsExecuted": "5", "actionsCreated": "5", "userTime": "1s", "systemTime": "1s"}]}}} +{"id": {"finished": {}}, "finished": {"exitCode": {"code": 0, "name": "SUCCESS; https://leak.example.com/token=abc"}, "finishTimeMillis": "1700000001000"}} +{"id": {"buildToolLogs": {}}, "buildToolLogs": {"log": [{"name": "critical path", "contents": "Q3JpdGljYWwgUGF0aDogMS4wMHMsIFJlbW90ZSAoMC4wJSBvZiB0aGUgdGltZSk6IFsKICAxLjAwcyBhY3Rpb24gJ1Rlc3RpbmcgLy86cGFzc190ZXN0IChzaGFyZCAwIG9mIDEpJyBzZWNyZXQ9aHVudGVyMgo="}, {"name": "elapsed time", "contents": "MS4w"}]}} +{"id": {"testSummary": {"label": "//pkg:x/etc/evil_test"}}, "testSummary": {"overallStatus": "PASSED; DEPLOY_TOKEN=s3cr3t-deploy-token-0001", "shardCount": 1, "attemptCount": 1, "totalNumCached": 0}} diff --git a/tools/bazel/testdata/ci_analytics/real/cold-bep.json b/tools/bazel/testdata/ci_analytics/real/cold-bep.json new file mode 100644 index 000000000..3ea8ad7af --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/real/cold-bep.json @@ -0,0 +1,12 @@ +{"id": {"started": {}}, "started": {"uuid": "ea882f0a-d70d-4e98-8471-874fa38dd26e", "startTimeMillis": "1791360355792", "buildToolVersion": "9.2.0", "command": "test"}} +{"id": {"testResult": {"label": "//:flaky_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "46", "status": "FAILED", "testAttemptDuration": "0.046s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.046s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:pass_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "43", "status": "PASSED", "testAttemptDuration": "0.043s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.043s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 2, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "45", "status": "PASSED", "testAttemptDuration": "0.045s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.045s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 3, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "49", "status": "PASSED", "testAttemptDuration": "0.049s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.049s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "56", "status": "PASSED", "testAttemptDuration": "0.056s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.056s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testSummary": {"label": "//:flaky_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "FAILED", "attemptCount": 1}} +{"id": {"testSummary": {"label": "//:pass_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "PASSED", "attemptCount": 1}} +{"id": {"testSummary": {"label": "//:sharded_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "PASSED", "shardCount": 3, "attemptCount": 1}} +{"id": {"buildFinished": {}}, "finished": {"finishTimeMillis": "1791360367926", "exitCode": {"name": "TESTS_FAILED", "code": 3}}} +{"id": {"buildToolLogs": {}}, "buildToolLogs": {"log": [{"name": "elapsed time", "contents": "MTIuMTM0MDAw"}, {"name": "critical path", "contents": "Q3JpdGljYWwgUGF0aDogMC4yM3MsIFJlbW90ZSAoMC4wMCUgb2YgdGhlIHRpbWUpOiBbcGFyc2U6IDAuMDAlLCBxdWV1ZTogMC4wMCUsIG5ldHdvcms6IDAuMDAlLCB1cGxvYWQ6IDAuMDAlLCBzZXR1cDogMC4wMCUsIHByb2Nlc3M6IDAuMDAlLCBmZXRjaDogMC4wMCUsIHJldHJ5OiAwLjAwJSwgcHJvY2Vzc091dHB1dHM6IDAuMDAlLCBvdGhlcjogMC4wMCUsIGlucHV0IGZpbGVzOiAwLCBpbnB1dCBieXRlczogMCwgbWVtb3J5IGJ5dGVzOiAwXQogIDAuMTFzIGFjdGlvbiAnVGVzdGluZyAvLzpzaGFyZGVkX3Rlc3QgKHNoYXJkIDMgb2YgMyknCiAgMC4wMHMgcnVuZmlsZXMgZm9yIC8vOnNoYXJkZWRfdGVzdAogIDAuMDdzIGFjdGlvbiAnTGlua2luZyBzaGFyZGVkX3Rlc3QnCiAgMC4wNXMgYWN0aW9uICdDb21waWxpbmcgbWFpbl9zaGFyZC5jJw=="}, {"name": "process stats", "contents": "MjUgcHJvY2Vzc2VzOiAxMyBpbnRlcm5hbCwgMTcgcHJvY2Vzc3dyYXBwZXItc2FuZGJveC4="}, {"name": "command.profile.json"}, {"name": "execution_log.binpb.zst"}]}} +{"id": {"buildMetrics": {}}, "buildMetrics": {"timingMetrics": {"cpuTimeInMs": "44660", "wallTimeInMs": "8862", "analysisPhaseTimeInMs": "2147", "executionPhaseTimeInMs": "2339", "actionsExecutionStartInMs": "6486", "criticalPathTime": "0.227630710s"}, "actionSummary": {"actionsCreated": "39", "actionsExecuted": "25", "runnerCount": [{"name": "total", "count": 25}, {"name": "internal", "count": 13}, {"name": "processwrapper-sandbox", "count": 17, "execKind": "Local"}], "actionCacheStatistics": {"misses": 25}, "actionData": [{"mnemonic": "TestRunner", "actionsExecuted": "5", "systemTime": "0.256s", "userTime": "0.063s", "actionsCreated": "5"}, {"mnemonic": "RepoMappingManifest", "actionsExecuted": "3", "actionsCreated": "4"}, {"mnemonic": "SymlinkTree", "actionsExecuted": "3", "actionsCreated": "4"}, {"mnemonic": "CppLink", "actionsExecuted": "3", "systemTime": "0.095s", "userTime": "0.064s", "actionsCreated": "3"}, {"mnemonic": "SourceSymlinkManifest", "actionsExecuted": "3", "actionsCreated": "4"}, {"mnemonic": "RunfilesTree", "actionsExecuted": "3", "actionsCreated": "4"}, {"mnemonic": "CppCompile", "actionsExecuted": "3", "systemTime": "0.036s", "userTime": "0.037s", "actionsCreated": "3"}, {"mnemonic": "BazelWorkspaceStatusAction", "actionsExecuted": "1", "actionsCreated": "1"}, {"mnemonic": "Genrule", "actionsExecuted": "1", "systemTime": "0.002s", "actionsCreated": "1"}, {"mnemonic": "TranslateBuildInfo", "actionsCreated": "2"}, {"mnemonic": "CcStrip", "actionsCreated": "3"}, {"mnemonic": "FileWrite", "actionsCreated": "3"}, {"mnemonic": "Symlink", "actionsCreated": "1"}, {"mnemonic": "ExecutableSymlink", "actionsCreated": "1"}]}, "packageMetrics": {"packagesLoaded": "90"}, "targetMetrics": {"targetsConfigured": "883"}, "networkMetrics": {"systemNetworkStats": {"bytesSent": "33058256", "bytesRecv": "134073050"}}}} diff --git a/tools/bazel/testdata/ci_analytics/real/cold-exec.binpb b/tools/bazel/testdata/ci_analytics/real/cold-exec.binpb new file mode 100644 index 000000000..24029aabc --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/real/cold-exec.binpb @@ -0,0 +1,549 @@ +86 +SHA-256_main"$ea882f0a-d70d-4e98-8471-874fa38dd26e€| +3external/bazel_tools/tools/genrule/genrule-setup.shE +@2138ccc4608e0a41291680587baced12bb80379b6924c3c4c2d9ef272f99139fâ2*rn +&bazel-out/k8-fastbuild/bin/gen_out.txtD +@5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03ˆ:… + /bin/bash +-c +osource external/bazel_tools/tools/genrule/genrule-setup.sh; echo hello > bazel-out/k8-fastbuild/bin/gen_out.txt$ +PATH/bin:/usr/bin:/usr/local/bin 2(: +//:gen_outBGenruleZprocesswrapper-sandboxhpx’# +€¤èB€¤è¢ íú—ÖÀ‡Ð�¨€ Ä§¢ +Yexternal/rules_cc++cc_configure_extension+local_config_cc/builtin_include_directory_pathsE +@8f7db3657f51747bb67c70c3f9ba2678ff87f1670e1a572c81aeef79af40c1deê•� +Gexternal/rules_cc++cc_configure_extension+local_config_cc/cc_wrapper.shE +@44de34ffa88565ff276fc5aa9971b262291db42cb486eb11cfece50454176d9b÷ +Ÿš +Qexternal/rules_cc++cc_configure_extension+local_config_cc/deps_scanner_wrapper.shE +@d273dd7612f1b110942c32c7e74046f27926fd576395d7fce0f225784fd21a1bî¢� +Texternal/rules_cc++cc_configure_extension+local_config_cc/validate_static_library.shE +@808d17e44d12a4e433a3f2d810760b641afce3714e44d350ba53ea7cf773dce8Õ +2*W S + main_fail.cD +@7b4bb8e34d20c4ec426e4770c2c8e808fea9f4c78893f9190b02c55e9f75e88b + +2"* ‰ „ +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.oE +@1b679cd82a4f53b030b31b689669f8e65c4c1e90c17f805078dd2a8edb7e0584Ð ˆ ƒ +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.dD +@082426df51ca02ea00b2382c7d7b1df16b38608090b3ce87f7c7634db459891cg„:� + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.d +I-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_fail.c +-o +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd +2( 2( : //:flaky_testB +CppCompileZprocesswrapper-sandboxhpx’# +€‡§B€‡§¢ ïú—Ö€øÅ©¨€€êW S + main_pass.cD +@2ad75d95660563887d8d3f1d0ae1dcf18c2379cbd83a5c72f5ab276351ee6949 +2"* ˆƒ +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.oE +@e1a5f327ee544bac718e414feb417ee9e04d39e9b1276e54e7e8ab02d29323b5Ð ‡‚ +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.dD +@2e8ca7b27e3234d03d277791f221f2662b3b7247afe26e933a69a3e4dd36e473f€:ý + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.d +H-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_pass.c +-o +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd 2(2(: //:pass_testB +CppCompileZprocesswrapper-sandboxhpx’# +À”ÞBÀ”Þ¢ ïú—Ö€øÅ©¨€€ÎYU + main_shard.cE +@7cfaebd87ccc933f629e4c77aad22d17d9372fb0845f98c99613e71df0028098ù +2"*Œ‡ +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.oE +@0d35f7d26f0b4af9e7b916c3905b7283fa2021551639d8a9ec079027d9a2c380° Œ‡ +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.dE +@f66abbbe9d483307c493d0cf1ab6809bf6aa64da7e155a7ba6e83ac66e976286®�:� + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.d +L-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_shard.c +-o +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd 2(2(://:sharded_testB +CppCompileZprocesswrapper-sandboxhpx’# +À¦ÒBÀ¦Ò¢ ïú—Ö€øÅ©¨€ � ‡‚ +:external/rules_cc+/cc/private/toolchain/build_interface_soD +@ca1c6c1ec88d12130f9d59bf6b1a9d0a81cf407c586c515717c2de63dc63a80bu�ˆ +?external/rules_cc+/cc/private/toolchain/link_dynamic_library.shE +@410914e519a1d309961ba77f4549a8c92da307ce1f596c10f8284da904b17a77� 2"* +2"* {w +.bazel-out/k8-fastbuild/bin/flaky_test-0.paramsE +@c577fe96cce31d4eea72ffbb6d7aa46945f0c984f907f3be440b2d575396cc06  +2"*rn +%bazel-out/k8-fastbuild/bin/flaky_testE +@674a99e83e5c4ed8beafe2af4d7b0ac254838e7f05e1c5c9cc9010cfe8a2dd78¨0ô:ñ + /usr/bin/gcc +/@bazel-out/k8-fastbuild/bin/flaky_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 2(: //:flaky_testBCppLinkZprocesswrapper-sandboxhpx’# +€óßB€óߢ ïú—ÖÀËú¿¨€ÀÅ +2"*zv +-bazel-out/k8-fastbuild/bin/pass_test-0.paramsE +@1c4343b244ba16563979291c4538e0c3eaad825d8f3c544062a69a6a939af671ž +2"*qm +$bazel-out/k8-fastbuild/bin/pass_testE +@d26a879186fed4dc742a5c000a7ab235b0a1009c0d291006150806c5c8015409¨0ò:ï + /usr/bin/gcc +.@bazel-out/k8-fastbuild/bin/pass_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 2(: //:pass_testBCppLinkZprocesswrapper-sandboxhpx’# +À÷œBÀ÷œ¢ ïú—ÖÀËú¿¨€ Ä + 2"*}!y +0bazel-out/k8-fastbuild/bin/sharded_test-0.paramsE +@bb9b5928db17b3f1215dc9e0405f5118783b5a7995020e6bc5fbd4df81f51d2e¥ +"2" *!t#p +'bazel-out/k8-fastbuild/bin/sharded_testE +@0a8dce773490fe6720cadc64b758164f2f15a9b1a38c0ac818ada7a0bbb84e67�4ø:õ + /usr/bin/gcc +1@bazel-out/k8-fastbuild/bin/sharded_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 "2(#://:sharded_testBCppLinkZprocesswrapper-sandboxhpx’# +€…ÔB€…Ô¢ ïú—ÖÀËú¿¨€€¡$2*}%Ry +-bazel-out/k8-fastbuild/bin/pass_test.runfiles$2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026az&v +-external/bazel_tools/tools/test/test-setup.shE +@49ba08927c3c556c52c6f771eaca362a0dbd1b6e19fd2667c61d92c33a32278aåt|'x +/external/bazel_tools/tools/test/generate-xml.shE +@368e50ceca617b237c60adf70105cf6e1d33427f232c78239a3e7c10a4d93ebfí' (2*%&')2*#€*R| +0bazel-out/k8-fastbuild/bin/sharded_test.runfiles)2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026a +2**&'�,Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855<-"8 +6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsL."H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs/2*~0Rz +.bazel-out/k8-fastbuild/bin/flaky_test.runfiles/2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026a 12*0&' 22**&'‚:ÿ +-external/bazel_tools/tools/test/test-setup.sh + ./pass_test> + JAVA_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/bin@ +PYTHON_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles= + RUNFILES_DIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./pass_testi + TEST_INFRASTRUCTURE_FAILURE_FILEEbazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failurek +TEST_LOGSPLITTER_OUTPUT_FILEKbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs] +TEST_PREMATURE_EXIT_FILEAbazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely + TEST_SIZEmedium< + TEST_SRCDIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles + TEST_TARGET //:pass_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/bbb994006a201e70ea51a76491bba66br +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSKbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONSj +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR?bazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifestU +TEST_UNDECLARED_OUTPUTS_DIR6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsl + TEST_UNDECLARED_OUTPUTS_MANIFESTHbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFESTc +TEST_UNUSED_RUNFILES_LOG_FILEBbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_logT +TEST_WARNINGS_OUTPUT_FILE7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings +TEST_WORKSPACE_main +TZUTCE +XML_OUTPUT_FILE2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml (24"2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml2C"Abazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely29"7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings2M"Kbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs2D"Bbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_log2G"Ebazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failure2(-2J"Hbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFEST2M"Kbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONS2P"Nbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONS.pb: //:pass_testB +TestRunnerZprocesswrapper-sandboxhpxˆà§’# +ÀÁÀBÀÁÀ¢ ïú—ÖÀ˜Ÿã¨€àÄÞ:Û +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely +TEST_SHARD_INDEX1^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/1615829df1fa3563387e50506a1f6692 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml +2D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely2(,2I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure2(.2Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpxˆà§’# +ÀʺBÀʺ¢ ïú—ÖÀ˜Ÿã¨€àÂ=3"9 +7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs�4Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855®:« +-external/bazel_tools/tools/test/test-setup.sh + ./flaky_test? + JAVA_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binA +PYTHON_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles> + RUNFILES_DIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./flaky_testj + TEST_INFRASTRUCTURE_FAILURE_FILEFbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failurel +TEST_LOGSPLITTER_OUTPUT_FILELbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs^ +TEST_PREMATURE_EXIT_FILEBbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely + TEST_SIZEmedium= + TEST_SRCDIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles + TEST_TARGET //:flaky_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/d778484882c0bff6932e3335e6656a87s +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSLbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONSk +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR@bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifestV +TEST_UNDECLARED_OUTPUTS_DIR7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputsm + TEST_UNDECLARED_OUTPUTS_MANIFESTIbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFESTd +TEST_UNUSED_RUNFILES_LOG_FILECbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_logU +TEST_WARNINGS_OUTPUT_FILE8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings +TEST_WORKSPACE_main +TZUTCF +XML_OUTPUT_FILE3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml 125"3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml2D"Bbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely2:"8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings2N"Lbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs2E"Cbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_log2H"Fbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failure2(32K"Ibazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFEST2N"Lbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONS2Q"Obazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONS.pb: //:flaky_testB +TestRunnerHR NON_ZERO_EXITZprocesswrapper-sandboxhpxˆà§’# +€Ï÷B€Ï÷¢ ïú—Ö€�Ü㨀àÄL5"H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputsÞ:Û +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely +TEST_SHARD_INDEX2^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/c0c31cf43ef7d650f7ec6ed08aec4b02 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml 22D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely2(42I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure2(52Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpxˆà§’# +ÀÜ®BÀÜ®¢ ïú—Ö€�Ü㨀ÀÄ 62**&'�7Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855L8"H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputsÞ:Û +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely +TEST_SHARD_INDEX0^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/baf5a5c95a10bc1fd7a56eb5c8e0a05a +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml 62D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely2(72I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure2(82Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpxˆà§’# +€üÙB€üÙ¢ ïú—Ö€�Ü㨀À€9| +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.logE +@f4f43b4414847462e830ccf2c6ed78b7f752d507751e9ac2f4edf797aee55ea0�:2*'9€;| +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xmlE +@c97b22d07a3c89431b8b3ecd40ca2e1ff7b17692616d90a4bf67f0606eae9cef²½:º +/external/bazel_tools/tools/test/generate-xml.sh +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.log +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml +0 +1? + JAVA_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binA +PYTHON_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles> + RUNFILES_DIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./flaky_testj + TEST_INFRASTRUCTURE_FAILURE_FILEFbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failurel +TEST_LOGSPLITTER_OUTPUT_FILELbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs + TEST_NAME //:flaky_test^ +TEST_PREMATURE_EXIT_FILEBbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely +TEST_SHARD_INDEX0 + TEST_SIZEmedium= + TEST_SRCDIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles + TEST_TARGET //:flaky_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/d778484882c0bff6932e3335e6656a87 +TEST_TOTAL_SHARDS0s +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSLbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONSk +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR@bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifestV +TEST_UNDECLARED_OUTPUTS_DIR7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputsm + TEST_UNDECLARED_OUTPUTS_MANIFESTIbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFESTd +TEST_UNUSED_RUNFILES_LOG_FILECbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_logU +TEST_WARNINGS_OUTPUT_FILE8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings +TEST_WORKSPACE_main +TZUTCF +XML_OUTPUT_FILE3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml :2(;: //:flaky_testB +TestRunnerZprocesswrapper-sandboxhpx’# +€ì¸ B€ì¸ ¢ ïú—ÖÀ¦íÿ¨€àÃ�<‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸ=2*'<�>‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xmlE +@cfe77d8780f07a6b93e2e76bb0806c9b5ec880e414605d03c0a9b92c7d0c57abšñ:î +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 3 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely +TEST_SHARD_INDEX2^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/c0c31cf43ef7d650f7ec6ed08aec4b02 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml =2(>://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpx’# +À‚ê BÀ‚ê ¢ ïú—ÖÀ¦íÿ¨€€Ã�?‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸ@2*'?�A‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xmlE +@8faa6c3e54a5d33b706bdd2917bf93f8e6522082b78cf016f23cc398558a7980šñ:î +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 2 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely +TEST_SHARD_INDEX1^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/1615829df1fa3563387e50506a1f6692 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml @2(A://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpx’# +€‡§B€‡§¢ ïú—ÖÀ¦íÿ¨€€ÁB{ +2bazel-out/k8-fastbuild/testlogs/pass_test/test.logE +@c4d97ed7504f0d4c742f57c5c7014ef33de0b8b077e90765a330ab231c2b09e5œC2*'BD{ +2bazel-out/k8-fastbuild/testlogs/pass_test/test.xmlE +@e69b20191cc399c99510a46ad7322a20bd7fe10c2ac70a01c4e73bf5ad48969cý©:¦ +/external/bazel_tools/tools/test/generate-xml.sh +2bazel-out/k8-fastbuild/testlogs/pass_test/test.log +2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml +0 +0> + JAVA_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/bin@ +PYTHON_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles= + RUNFILES_DIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./pass_testi + TEST_INFRASTRUCTURE_FAILURE_FILEEbazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failurek +TEST_LOGSPLITTER_OUTPUT_FILEKbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs + TEST_NAME //:pass_test] +TEST_PREMATURE_EXIT_FILEAbazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely +TEST_SHARD_INDEX0 + TEST_SIZEmedium< + TEST_SRCDIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles + TEST_TARGET //:pass_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/bbb994006a201e70ea51a76491bba66b +TEST_TOTAL_SHARDS0r +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSKbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONSj +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR?bazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifestU +TEST_UNDECLARED_OUTPUTS_DIR6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsl + TEST_UNDECLARED_OUTPUTS_MANIFESTHbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFESTc +TEST_UNUSED_RUNFILES_LOG_FILEBbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_logT +TEST_WARNINGS_OUTPUT_FILE7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings +TEST_WORKSPACE_main +TZUTCE +XML_OUTPUT_FILE2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml C2(D: //:pass_testB +TestRunnerZprocesswrapper-sandboxhpx’# +À‹äBÀ‹ä¢ ïú—ÖÀ¦íÿ¨€ÀÃ�E‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸF2*'E�G‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xmlE +@da3be6ada8aa77a9ec8381222bb8358fcd7300c46bc8318e05b7cd792a471f60šñ:î +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 1 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely +TEST_SHARD_INDEX0^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/baf5a5c95a10bc1fd7a56eb5c8e0a05a +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml F2(G://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpx’# +€�¡B€�¡¢ ïú—ÖÀ¸á�¨€àà \ No newline at end of file diff --git a/tools/bazel/testdata/ci_analytics/real/cold-profile.json b/tools/bazel/testdata/ci_analytics/real/cold-profile.json new file mode 100644 index 000000000..96a1649ed --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/real/cold-profile.json @@ -0,0 +1,170 @@ +{"traceEvents":[ +{"cat": "Fetching repository", "name": "@@bazel_tools", "ph": "X", "ts": 658997, "dur": 1250, "pid": 1, "tid": 172}, +{"cat": "Fetching repository", "name": "@@bazel_tools", "ph": "X", "ts": 660759, "dur": 1022, "pid": 1, "tid": 172}, +{"cat": "Fetching repository", "name": "@@bazel_tools", "ph": "X", "ts": 662835, "dur": 371, "pid": 1, "tid": 172}, +{"cat": "Fetching repository", "name": "@@rules_cc+", "ph": "X", "ts": 3904359, "dur": 32, "pid": 1, "tid": 316}, +{"cat": "Fetching repository", "name": "@@rules_cc+", "ph": "X", "ts": 3953422, "dur": 10926, "pid": 1, "tid": 316}, +{"cat": "Fetching repository", "name": "@@rules_cc+", "ph": "X", "ts": 3964567, "dur": 536104, "pid": 1, "tid": 316}, +{"cat": "Fetching repository", "name": "@@rules_cc+", "ph": "X", "ts": 4501714, "dur": 461, "pid": 1, "tid": 316}, +{"cat": "Fetching repository", "name": "@@rules_cc++compatibility_proxy+cc_compatibility_proxy", "ph": "X", "ts": 4515525, "dur": 24, "pid": 1, "tid": 327}, +{"cat": "Fetching repository", "name": "@@bazel_features+", "ph": "X", "ts": 4529433, "dur": 37, "pid": 1, "tid": 269}, +{"cat": "Fetching repository", "name": "@@bazel_features+", "ph": "X", "ts": 4530239, "dur": 111971, "pid": 1, "tid": 269}, +{"cat": "Fetching repository", "name": "@@bazel_features+", "ph": "X", "ts": 4643081, "dur": 286, "pid": 1, "tid": 269}, +{"cat": "Fetching repository", "name": "@@bazel_features++version_extension+bazel_features_globals", "ph": "X", "ts": 4648893, "dur": 19, "pid": 1, "tid": 290}, +{"cat": "Fetching repository", "name": "@@bazel_features++version_extension+bazel_features_version", "ph": "X", "ts": 4649521, "dur": 14, "pid": 1, "tid": 280}, +{"cat": "Fetching repository", "name": "@@bazel_skylib+", "ph": "X", "ts": 4651025, "dur": 16, "pid": 1, "tid": 324}, +{"cat": "Fetching repository", "name": "@@bazel_skylib+", "ph": "X", "ts": 4651819, "dur": 171321, "pid": 1, "tid": 324}, +{"cat": "Fetching repository", "name": "@@bazel_skylib+", "ph": "X", "ts": 4823901, "dur": 258, "pid": 1, "tid": 324}, +{"cat": "Fetching repository", "name": "@@bazel_features++version_extension+bazel_features_version", "ph": "X", "ts": 4869764, "dur": 6204, "pid": 1, "tid": 324}, +{"cat": "Fetching repository", "name": "@@bazel_features++version_extension+bazel_features_globals", "ph": "X", "ts": 4869967, "dur": 16837, "pid": 1, "tid": 280}, +{"cat": "Fetching repository", "name": "@@rules_cc++compatibility_proxy+cc_compatibility_proxy", "ph": "X", "ts": 4901925, "dur": 2388, "pid": 1, "tid": 269}, +{"cat": "Fetching repository", "name": "@@protobuf+", "ph": "X", "ts": 4922218, "dur": 18, "pid": 1, "tid": 425}, +{"cat": "Fetching repository", "name": "@@protobuf+", "ph": "X", "ts": 4923661, "dur": 848913, "pid": 1, "tid": 425}, +{"cat": "Fetching repository", "name": "@@protobuf+", "ph": "X", "ts": 5772986, "dur": 211, "pid": 1, "tid": 425}, +{"cat": "Fetching repository", "name": "@@platforms", "ph": "X", "ts": 6054930, "dur": 13, "pid": 1, "tid": 461}, +{"cat": "Fetching repository", "name": "@@platforms", "ph": "X", "ts": 6055425, "dur": 41544, "pid": 1, "tid": 461}, +{"cat": "Fetching repository", "name": "@@platforms", "ph": "X", "ts": 6097222, "dur": 159, "pid": 1, "tid": 461}, +{"cat": "Fetching repository", "name": "@@platforms+host_platform+host_platform", "ph": "X", "ts": 6101665, "dur": 23, "pid": 1, "tid": 463}, +{"cat": "Fetching repository", "name": "@@platforms+host_platform+host_platform", "ph": "X", "ts": 6105780, "dur": 2496, "pid": 1, "tid": 463}, +{"cat": "Fetching repository", "name": "@@rules_swift++non_module_deps+build_bazel_rules_swift_local_config", "ph": "X", "ts": 6414717, "dur": 19, "pid": 1, "tid": 666}, +{"cat": "Fetching repository", "name": "@@rules_swift+", "ph": "X", "ts": 6415310, "dur": 11, "pid": 1, "tid": 666}, +{"cat": "Fetching repository", "name": "@@bazel_tools+winsdk_configure+local_config_winsdk", "ph": "X", "ts": 6417309, "dur": 27, "pid": 1, "tid": 698}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc_toolchains", "ph": "X", "ts": 6418768, "dur": 29, "pid": 1, "tid": 681}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+local_jdk", "ph": "X", "ts": 6426581, "dur": 23, "pid": 1, "tid": 717}, +{"cat": "Fetching repository", "name": "@@rules_java+", "ph": "X", "ts": 6428070, "dur": 19, "pid": 1, "tid": 697}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc_toolchains", "ph": "X", "ts": 6434044, "dur": 271, "pid": 1, "tid": 807}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_toolchain_config_repo", "ph": "X", "ts": 6435224, "dur": 21, "pid": 1, "tid": 746}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_macos_toolchain_config_repo", "ph": "X", "ts": 6436035, "dur": 25, "pid": 1, "tid": 748}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6436767, "dur": 20, "pid": 1, "tid": 752}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6437255, "dur": 25, "pid": 1, "tid": 832}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6438672, "dur": 36, "pid": 1, "tid": 715}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_toolchain_config_repo", "ph": "X", "ts": 6439596, "dur": 23, "pid": 1, "tid": 682}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_windows_toolchain_config_repo", "ph": "X", "ts": 6439942, "dur": 20, "pid": 1, "tid": 763}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6440519, "dur": 19, "pid": 1, "tid": 759}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_toolchain_config_repo", "ph": "X", "ts": 6440736, "dur": 17, "pid": 1, "tid": 683}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6442806, "dur": 16, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6442996, "dur": 5, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_win_toolchain_config_repo", "ph": "X", "ts": 6443159, "dur": 5, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_toolchain_config_repo", "ph": "X", "ts": 6443317, "dur": 4, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6443305, "dur": 21, "pid": 1, "tid": 722}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6443518, "dur": 5, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6443604, "dur": 24, "pid": 1, "tid": 696}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_riscv64_toolchain_config_repo", "ph": "X", "ts": 6443760, "dur": 6, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6443771, "dur": 12, "pid": 1, "tid": 722}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_macos_toolchain_config_repo", "ph": "X", "ts": 6443908, "dur": 7, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6444086, "dur": 12, "pid": 1, "tid": 696}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6444184, "dur": 10, "pid": 1, "tid": 722}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_win_toolchain_config_repo", "ph": "X", "ts": 6444205, "dur": 9, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc_toolchains", "ph": "X", "ts": 6438398, "dur": 6035, "pid": 1, "tid": 838}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_linux_toolchain_config_repo", "ph": "X", "ts": 6445238, "dur": 9, "pid": 1, "tid": 778}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6445226, "dur": 16, "pid": 1, "tid": 685}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_macos_toolchain_config_repo", "ph": "X", "ts": 6445226, "dur": 16, "pid": 1, "tid": 777}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6445315, "dur": 12, "pid": 1, "tid": 700}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_macos_toolchain_config_repo", "ph": "X", "ts": 6445593, "dur": 11, "pid": 1, "tid": 723}, +{"cat": "Fetching repository", "name": "@@rules_python++python+pythons_hub", "ph": "X", "ts": 6445625, "dur": 12, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6445624, "dur": 14, "pid": 1, "tid": 696}, +{"cat": "Fetching repository", "name": "@@apple_support++apple_cc_configure_extension+local_config_apple_cc_toolchains", "ph": "X", "ts": 6445646, "dur": 7, "pid": 1, "tid": 777}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6445668, "dur": 13, "pid": 1, "tid": 778}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_win_toolchain_config_repo", "ph": "X", "ts": 6445685, "dur": 11, "pid": 1, "tid": 722}, +{"cat": "Fetching repository", "name": "@@rules_python++uv+uv", "ph": "X", "ts": 6445761, "dur": 7, "pid": 1, "tid": 685}, +{"cat": "Fetching repository", "name": "@@rules_kotlin+", "ph": "X", "ts": 6445765, "dur": 7, "pid": 1, "tid": 700}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_win_toolchain_config_repo", "ph": "X", "ts": 6445728, "dur": 22, "pid": 1, "tid": 724}, +{"cat": "Fetching repository", "name": "@@rules_shell++sh_configure+local_config_shell", "ph": "X", "ts": 6445786, "dur": 25, "pid": 1, "tid": 780}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6446119, "dur": 13, "pid": 1, "tid": 782}, +{"cat": "Fetching repository", "name": "@@rules_shell+", "ph": "X", "ts": 6446992, "dur": 12, "pid": 1, "tid": 727}, +{"cat": "Fetching repository", "name": "@@rules_python+", "ph": "X", "ts": 6447080, "dur": 14, "pid": 1, "tid": 773}, +{"cat": "Fetching repository", "name": "@@apple_support+", "ph": "X", "ts": 6447500, "dur": 10, "pid": 1, "tid": 779}, +{"cat": "Fetching repository", "name": "@@bazel_tools+winsdk_configure+local_config_winsdk", "ph": "X", "ts": 6451198, "dur": 356, "pid": 1, "tid": 703}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6456645, "dur": 18, "pid": 1, "tid": 706}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6458223, "dur": 12, "pid": 1, "tid": 689}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6460249, "dur": 13, "pid": 1, "tid": 747}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_macos_toolchain_config_repo", "ph": "X", "ts": 6467768, "dur": 21, "pid": 1, "tid": 845}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6470984, "dur": 21, "pid": 1, "tid": 765}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc_toolchains", "ph": "X", "ts": 6445781, "dur": 31777, "pid": 1, "tid": 838}, +{"cat": "Fetching repository", "name": "@@bazel_tools+winsdk_configure+local_config_winsdk", "ph": "X", "ts": 6451781, "dur": 26346, "pid": 1, "tid": 734}, +{"cat": "Fetching repository", "name": "@@rules_java+", "ph": "X", "ts": 6436289, "dur": 112074, "pid": 1, "tid": 687}, +{"cat": "Fetching repository", "name": "@@rules_java+", "ph": "X", "ts": 6548753, "dur": 150, "pid": 1, "tid": 687}, +{"cat": "Fetching repository", "name": "@@rules_java++compatibility_proxy+compatibility_proxy", "ph": "X", "ts": 6558073, "dur": 15, "pid": 1, "tid": 731}, +{"cat": "Fetching repository", "name": "@@rules_java++compatibility_proxy+compatibility_proxy", "ph": "X", "ts": 6562917, "dur": 2676, "pid": 1, "tid": 770}, +{"cat": "Fetching repository", "name": "@@apple_support+", "ph": "X", "ts": 6447998, "dur": 173552, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@apple_support+", "ph": "X", "ts": 6621865, "dur": 374, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_swift+", "ph": "X", "ts": 6416123, "dur": 208585, "pid": 1, "tid": 666}, +{"cat": "Fetching repository", "name": "@@rules_swift+", "ph": "X", "ts": 6624873, "dur": 537, "pid": 1, "tid": 666}, +{"cat": "Fetching repository", "name": "@@apple_support++apple_cc_configure_extension+local_config_apple_cc_toolchains", "ph": "X", "ts": 6633898, "dur": 156, "pid": 1, "tid": 694}, +{"cat": "Fetching repository", "name": "@@apple_support++apple_cc_configure_extension+local_config_apple_cc_toolchains", "ph": "X", "ts": 6634191, "dur": 3994, "pid": 1, "tid": 860}, +{"cat": "Fetching repository", "name": "@@apple_support++apple_cc_configure_extension+local_config_apple_cc_toolchains", "ph": "X", "ts": 6638675, "dur": 878, "pid": 1, "tid": 860}, +{"cat": "Fetching repository", "name": "@@rules_swift++non_module_deps+build_bazel_rules_swift_local_config", "ph": "X", "ts": 6639839, "dur": 275, "pid": 1, "tid": 666}, +{"cat": "Fetching repository", "name": "@@rules_swift++non_module_deps+build_bazel_rules_swift_local_config", "ph": "X", "ts": 6640214, "dur": 4287, "pid": 1, "tid": 860}, +{"cat": "Fetching repository", "name": "@@rules_shell+", "ph": "X", "ts": 6447498, "dur": 225173, "pid": 1, "tid": 782}, +{"cat": "Fetching repository", "name": "@@rules_shell+", "ph": "X", "ts": 6673022, "dur": 169, "pid": 1, "tid": 782}, +{"cat": "Fetching repository", "name": "@@rules_shell++sh_configure+local_config_shell", "ph": "X", "ts": 6679879, "dur": 199, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_shell++sh_configure+local_config_shell", "ph": "X", "ts": 6680241, "dur": 2662, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+local_jdk", "ph": "X", "ts": 6685054, "dur": 379, "pid": 1, "tid": 854}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_win_toolchain_config_repo", "ph": "X", "ts": 6685323, "dur": 5559, "pid": 1, "tid": 683}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685099, "dur": 6503, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_toolchain_config_repo", "ph": "X", "ts": 6684997, "dur": 6803, "pid": 1, "tid": 732}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_macos_toolchain_config_repo", "ph": "X", "ts": 6685392, "dur": 6884, "pid": 1, "tid": 665}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6685115, "dur": 7208, "pid": 1, "tid": 856}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6685367, "dur": 7550, "pid": 1, "tid": 670}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_toolchain_config_repo", "ph": "X", "ts": 6685324, "dur": 7597, "pid": 1, "tid": 768}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6684990, "dur": 8176, "pid": 1, "tid": 782}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685222, "dur": 8048, "pid": 1, "tid": 766}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6685348, "dur": 8862, "pid": 1, "tid": 790}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685310, "dur": 8952, "pid": 1, "tid": 857}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685220, "dur": 9481, "pid": 1, "tid": 682}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6685182, "dur": 9900, "pid": 1, "tid": 719}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685359, "dur": 9680, "pid": 1, "tid": 865}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6685461, "dur": 9896, "pid": 1, "tid": 764}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_win_toolchain_config_repo", "ph": "X", "ts": 6685455, "dur": 11082, "pid": 1, "tid": 759}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6685164, "dur": 11410, "pid": 1, "tid": 762}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6685111, "dur": 12321, "pid": 1, "tid": 869}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_win_toolchain_config_repo", "ph": "X", "ts": 6685202, "dur": 12282, "pid": 1, "tid": 851}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_toolchain_config_repo", "ph": "X", "ts": 6685107, "dur": 12890, "pid": 1, "tid": 767}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685095, "dur": 13732, "pid": 1, "tid": 866}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+local_jdk", "ph": "X", "ts": 6685588, "dur": 13336, "pid": 1, "tid": 849}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_macos_toolchain_config_repo", "ph": "X", "ts": 6685509, "dur": 13589, "pid": 1, "tid": 716}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_macos_toolchain_config_repo", "ph": "X", "ts": 6685221, "dur": 13932, "pid": 1, "tid": 821}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_macos_toolchain_config_repo", "ph": "X", "ts": 6685051, "dur": 16157, "pid": 1, "tid": 701}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6685219, "dur": 16363, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6686285, "dur": 15952, "pid": 1, "tid": 850}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_linux_toolchain_config_repo", "ph": "X", "ts": 6685288, "dur": 17582, "pid": 1, "tid": 731}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6685099, "dur": 17808, "pid": 1, "tid": 714}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6685225, "dur": 17968, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685222, "dur": 18683, "pid": 1, "tid": 863}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_windows_toolchain_config_repo", "ph": "X", "ts": 6685092, "dur": 19211, "pid": 1, "tid": 771}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_win_toolchain_config_repo", "ph": "X", "ts": 6685240, "dur": 19807, "pid": 1, "tid": 686}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_toolchain_config_repo", "ph": "X", "ts": 6685223, "dur": 19891, "pid": 1, "tid": 867}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_riscv64_toolchain_config_repo", "ph": "X", "ts": 6685270, "dur": 19975, "pid": 1, "tid": 770}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685383, "dur": 20945, "pid": 1, "tid": 734}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685358, "dur": 20992, "pid": 1, "tid": 852}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_macos_toolchain_config_repo", "ph": "X", "ts": 6685366, "dur": 21318, "pid": 1, "tid": 838}, +{"cat": "Fetching repository", "name": "@@rules_kotlin+", "ph": "X", "ts": 6446582, "dur": 609243, "pid": 1, "tid": 700}, +{"cat": "Fetching repository", "name": "@@rules_kotlin+", "ph": "X", "ts": 7056087, "dur": 135, "pid": 1, "tid": 700}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7066480, "dur": 20, "pid": 1, "tid": 769}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7073352, "dur": 1303, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7074884, "dur": 448, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7076116, "dur": 215, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7076575, "dur": 912, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7077639, "dur": 562, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_python+", "ph": "X", "ts": 6447628, "dur": 696061, "pid": 1, "tid": 784}, +{"cat": "Fetching repository", "name": "@@rules_python+", "ph": "X", "ts": 7144348, "dur": 234, "pid": 1, "tid": 784}, +{"cat": "Fetching repository", "name": "@@rules_python++uv+uv", "ph": "X", "ts": 7158611, "dur": 2966, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_python++python+pythons_hub", "ph": "X", "ts": 7251169, "dur": 7712, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_license+", "ph": "X", "ts": 7289289, "dur": 19, "pid": 1, "tid": 694}, +{"cat": "Fetching repository", "name": "@@rules_python++config+rules_python_internal", "ph": "X", "ts": 7299107, "dur": 16, "pid": 1, "tid": 860}, +{"cat": "Fetching repository", "name": "@@rules_python++config+rules_python_internal", "ph": "X", "ts": 7306163, "dur": 214, "pid": 1, "tid": 734}, +{"cat": "Fetching repository", "name": "@@rules_python++config+rules_python_internal", "ph": "X", "ts": 7306477, "dur": 1491, "pid": 1, "tid": 734}, +{"cat": "Fetching repository", "name": "@@rules_python++config+rules_python_internal", "ph": "X", "ts": 7308087, "dur": 2631, "pid": 1, "tid": 734}, +{"cat": "Fetching repository", "name": "@@rules_license+", "ph": "X", "ts": 7289673, "dur": 44756, "pid": 1, "tid": 694}, +{"cat": "Fetching repository", "name": "@@rules_license+", "ph": "X", "ts": 7334636, "dur": 163, "pid": 1, "tid": 694}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7430613, "dur": 18, "pid": 1, "tid": 784}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7430912, "dur": 285, "pid": 1, "tid": 784}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7431527, "dur": 37249, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7469049, "dur": 688, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7469907, "dur": 462, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7470561, "dur": 714698, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 8185888, "dur": 1894, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@bazel_tools+xcode_configure_extension+local_config_xcode", "ph": "X", "ts": 8206177, "dur": 16, "pid": 1, "tid": 865}, +{"cat": "Fetching repository", "name": "@@bazel_tools+xcode_configure_extension+local_config_xcode", "ph": "X", "ts": 8207846, "dur": 202, "pid": 1, "tid": 858}, +{"cat": "Fetching repository", "name": "@@bazel_tools+xcode_configure_extension+local_config_xcode", "ph": "X", "ts": 8208145, "dur": 1651, "pid": 1, "tid": 858} +]} diff --git a/tools/bazel/testdata/ci_analytics/real/postclean-bep.json b/tools/bazel/testdata/ci_analytics/real/postclean-bep.json new file mode 100644 index 000000000..a7d90d64e --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/real/postclean-bep.json @@ -0,0 +1,12 @@ +{"id": {"started": {}}, "started": {"uuid": "f2369dd1-2e00-4af2-8147-111452c04cce", "startTimeMillis": "1791360370092", "buildToolVersion": "9.2.0", "command": "test"}} +{"id": {"testResult": {"label": "//:pass_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "41", "status": "PASSED", "testAttemptDuration": "0.041s", "executionInfo": {"strategy": "disk cache hit", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0.001s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testSummary": {"label": "//:pass_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "PASSED", "totalNumCached": 1, "attemptCount": 1}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "44", "status": "PASSED", "testAttemptDuration": "0.044s", "executionInfo": {"strategy": "disk cache hit", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0.001s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 2, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "45", "status": "PASSED", "testAttemptDuration": "0.045s", "executionInfo": {"strategy": "disk cache hit", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0.001s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 3, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "45", "status": "PASSED", "testAttemptDuration": "0.045s", "executionInfo": {"strategy": "disk cache hit", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0.001s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testSummary": {"label": "//:sharded_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "PASSED", "totalNumCached": 3, "shardCount": 3, "attemptCount": 1}} +{"id": {"testResult": {"label": "//:flaky_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "36", "status": "FAILED", "testAttemptDuration": "0.036s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.036s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testSummary": {"label": "//:flaky_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "FAILED", "attemptCount": 1}} +{"id": {"buildFinished": {}}, "finished": {"finishTimeMillis": "1791360370833", "exitCode": {"name": "TESTS_FAILED", "code": 3}}} +{"id": {"buildToolLogs": {}}, "buildToolLogs": {"log": [{"name": "elapsed time", "contents": "MC43NDEwMDA="}, {"name": "critical path", "contents": "Q3JpdGljYWwgUGF0aDogMC4xMXMsIFJlbW90ZSAoMC4wMCUgb2YgdGhlIHRpbWUpOiBbcGFyc2U6IDAuMDAlLCBxdWV1ZTogMC4wMCUsIG5ldHdvcms6IDAuMDAlLCB1cGxvYWQ6IDAuMDAlLCBzZXR1cDogMC4wMCUsIHByb2Nlc3M6IDAuMDAlLCBmZXRjaDogMC4wMCUsIHJldHJ5OiAwLjAwJSwgcHJvY2Vzc091dHB1dHM6IDAuMDAlLCBvdGhlcjogMC4wMCUsIGlucHV0IGZpbGVzOiAwLCBpbnB1dCBieXRlczogMCwgbWVtb3J5IGJ5dGVzOiAwXQogIDAuMDhzIGFjdGlvbiAnVGVzdGluZyAvLzpmbGFreV90ZXN0JwogIDAuMDBzIHJ1bmZpbGVzIGZvciAvLzpmbGFreV90ZXN0CiAgMC4wMXMgYWN0aW9uICdMaW5raW5nIGZsYWt5X3Rlc3QnCiAgMC4wMXMgYWN0aW9uICdDb21waWxpbmcgbWFpbl9mYWlsLmMn"}, {"name": "process stats", "contents": "MjUgcHJvY2Vzc2VzOiAxNiBkaXNrIGNhY2hlIGhpdCwgMTMgaW50ZXJuYWwsIDEgcHJvY2Vzc3dyYXBwZXItc2FuZGJveC4="}, {"name": "command.profile.gz"}, {"name": "execution_log.binpb.zst"}]}} +{"id": {"buildMetrics": {}}, "buildMetrics": {"timingMetrics": {"cpuTimeInMs": "11020", "wallTimeInMs": "733", "analysisPhaseTimeInMs": "332", "executionPhaseTimeInMs": "386", "actionsExecutionStartInMs": "336", "criticalPathTime": "0.111170468s"}, "actionSummary": {"actionsCreated": "39", "actionsExecuted": "25", "runnerCount": [{"name": "total", "count": 25}, {"name": "disk cache hit", "count": 16, "execKind": "Remote"}, {"name": "internal", "count": 13}, {"name": "processwrapper-sandbox", "count": 1, "execKind": "Local"}], "actionCacheStatistics": {"misses": 25}, "actionData": [{"mnemonic": "TestRunner", "actionsExecuted": "5", "systemTime": "0.032s", "userTime": "0.004s", "actionsCreated": "15"}, {"mnemonic": "RepoMappingManifest", "actionsExecuted": "3", "actionsCreated": "12"}, {"mnemonic": "SymlinkTree", "actionsExecuted": "3", "actionsCreated": "12"}, {"mnemonic": "CppLink", "actionsExecuted": "3", "actionsCreated": "9"}, {"mnemonic": "SourceSymlinkManifest", "actionsExecuted": "3", "actionsCreated": "12"}, {"mnemonic": "RunfilesTree", "actionsExecuted": "3", "actionsCreated": "12"}, {"mnemonic": "CppCompile", "actionsExecuted": "3", "actionsCreated": "9"}, {"mnemonic": "BazelWorkspaceStatusAction", "actionsExecuted": "1", "actionsCreated": "2"}, {"mnemonic": "Genrule", "actionsExecuted": "1", "actionsCreated": "3"}, {"mnemonic": "TranslateBuildInfo", "actionsCreated": "6"}, {"mnemonic": "CcStrip", "actionsCreated": "9"}, {"mnemonic": "FileWrite", "actionsCreated": "9"}, {"mnemonic": "Symlink", "actionsCreated": "3"}, {"mnemonic": "ExecutableSymlink", "actionsCreated": "3"}]}, "packageMetrics": {"packagesLoaded": "90"}, "targetMetrics": {"targetsConfigured": "883"}, "networkMetrics": {"systemNetworkStats": {"bytesSent": "43171986", "bytesRecv": "146569473"}}}} diff --git a/tools/bazel/testdata/ci_analytics/real/postclean-exec.binpb b/tools/bazel/testdata/ci_analytics/real/postclean-exec.binpb new file mode 100644 index 000000000..13ec2a044 --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/real/postclean-exec.binpb @@ -0,0 +1,567 @@ +86 +SHA-256_main"$f2369dd1-2e00-4af2-8147-111452c04cce€| +3external/bazel_tools/tools/genrule/genrule-setup.shE +@2138ccc4608e0a41291680587baced12bb80379b6924c3c4c2d9ef272f99139fâ2*rn +&bazel-out/k8-fastbuild/bin/gen_out.txtD +@5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03È:Å + /bin/bash +-c +osource external/bazel_tools/tools/genrule/genrule-setup.sh; echo hello > bazel-out/k8-fastbuild/bin/gen_out.txt$ +PATH/bin:/usr/bin:/usr/local/bin 2(: +//:gen_outBGenruleZdisk cache hit`hpx‚E +@7bc5e991e8d71596d1042255f093ee1907c4614860ba7e1fe09ab6baf546c89b‘’! +€¤è"€’ôX† +`¢ ñú—Ö€µ·I§¢ +Yexternal/rules_cc++cc_configure_extension+local_config_cc/builtin_include_directory_pathsE +@8f7db3657f51747bb67c70c3f9ba2678ff87f1670e1a572c81aeef79af40c1deê•� +Gexternal/rules_cc++cc_configure_extension+local_config_cc/cc_wrapper.shE +@44de34ffa88565ff276fc5aa9971b262291db42cb486eb11cfece50454176d9b÷ +Ÿš +Qexternal/rules_cc++cc_configure_extension+local_config_cc/deps_scanner_wrapper.shE +@d273dd7612f1b110942c32c7e74046f27926fd576395d7fce0f225784fd21a1bî¢� +Texternal/rules_cc++cc_configure_extension+local_config_cc/validate_static_library.shE +@808d17e44d12a4e433a3f2d810760b641afce3714e44d350ba53ea7cf773dce8Õ +2*W S + main_fail.cD +@7b4bb8e34d20c4ec426e4770c2c8e808fea9f4c78893f9190b02c55e9f75e88b + +2"* ‰ „ +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.oE +@1b679cd82a4f53b030b31b689669f8e65c4c1e90c17f805078dd2a8edb7e0584Ð W S + main_pass.cD +@2ad75d95660563887d8d3f1d0ae1dcf18c2379cbd83a5c72f5ab276351ee6949 + 2"* ˆƒ +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.dD +@082426df51ca02ea00b2382c7d7b1df16b38608090b3ce87f7c7634db459891cgˆƒ +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.oE +@e1a5f327ee544bac718e414feb417ee9e04d39e9b1276e54e7e8ab02d29323b5Ð Å: + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.d +I-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_fail.c +-o +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd +2( 2(: //:flaky_testB +CppCompileZdisk cache hit`hpx‚E +@7db0318384af7d25cd261e60e27e11b0d098c735e707cba5d8236048e5bce944’’" +À¨¥"ÀŸ«Xö%`¢ ñú—ÖÀá‹­‡‚ +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.dD +@2e8ca7b27e3234d03d277791f221f2662b3b7247afe26e933a69a3e4dd36e473fÁ:¾ + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.d +H-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_pass.c +-o +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd 2(2(: //:pass_testB +CppCompileZdisk cache hit`hpx‚E +@edafc7f3d7eb8e1f73b94703ab903cd05556b98b31a5922b2439528943cbc311’’" +À¨¥"€›îXö%`¢ ñú—ÖÀØ‘¬‡‚ +:external/rules_cc+/cc/private/toolchain/build_interface_soD +@ca1c6c1ec88d12130f9d59bf6b1a9d0a81cf407c586c515717c2de63dc63a80bu�ˆ +?external/rules_cc+/cc/private/toolchain/link_dynamic_library.shE +@410914e519a1d309961ba77f4549a8c92da307ce1f596c10f8284da904b17a77� 2"* +2"* {w +.bazel-out/k8-fastbuild/bin/flaky_test-0.paramsE +@c577fe96cce31d4eea72ffbb6d7aa46945f0c984f907f3be440b2d575396cc06  +2"* +2"*zv +-bazel-out/k8-fastbuild/bin/pass_test-0.paramsE +@1c4343b244ba16563979291c4538e0c3eaad825d8f3c544062a69a6a939af671ž +2"*rn +%bazel-out/k8-fastbuild/bin/flaky_testE +@674a99e83e5c4ed8beafe2af4d7b0ac254838e7f05e1c5c9cc9010cfe8a2dd78¨0qm +$bazel-out/k8-fastbuild/bin/pass_testE +@d26a879186fed4dc742a5c000a7ab235b0a1009c0d291006150806c5c8015409¨0¬:© + /usr/bin/gcc +.@bazel-out/k8-fastbuild/bin/pass_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 2(: //:pass_testBCppLinkZdisk cache hit`hpx‚E +@40a597b9bccac093484d0c5f4e3afa0f6cc312bd363a2dac74c8ef6e269d344d’’ +À±ŸXÝZ`¢ ñú—Ö€£ÌÁ®:« + /usr/bin/gcc +/@bazel-out/k8-fastbuild/bin/flaky_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 2(: //:flaky_testBCppLinkZdisk cache hit`hpx‚E +@4894cc5cc7eba8d833a50277c20d3f8ca3e8d632ad4de6ec97ba0d469d36c578’’ +€­âXáZ`¢ ñú—Ö€µÀÃYU + main_shard.cE +@7cfaebd87ccc933f629e4c77aad22d17d9372fb0845f98c99613e71df0028098ù +2"*Œ‡ +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.oE +@0d35f7d26f0b4af9e7b916c3905b7283fa2021551639d8a9ec079027d9a2c380° Œ‡ +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.dE +@f66abbbe9d483307c493d0cf1ab6809bf6aa64da7e155a7ba6e83ac66e976286®Ê:Ç + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.d +L-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_shard.c +-o +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd 2(2(://:sharded_testB +CppCompileZdisk cache hit`hpx‚E +@ecb953f2ab45730fa4a7a8d72dac46d1a42af9563bdf7a599c6d3fca1f3add26’’ +€­âXÔ'`¢ ñú—ÖÀž�Á 2*}!Ry +-bazel-out/k8-fastbuild/bin/pass_test.runfiles 2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026az"v +-external/bazel_tools/tools/test/test-setup.shE +@49ba08927c3c556c52c6f771eaca362a0dbd1b6e19fd2667c61d92c33a32278aåt|#x +/external/bazel_tools/tools/test/generate-xml.shE +@368e50ceca617b237c60adf70105cf6e1d33427f232c78239a3e7c10a4d93ebfí' $2*!"#<%"8 +6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsÃ:À +-external/bazel_tools/tools/test/test-setup.sh + ./pass_test> + JAVA_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/bin@ +PYTHON_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles= + RUNFILES_DIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./pass_testi + TEST_INFRASTRUCTURE_FAILURE_FILEEbazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failurek +TEST_LOGSPLITTER_OUTPUT_FILEKbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs] +TEST_PREMATURE_EXIT_FILEAbazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely + TEST_SIZEmedium< + TEST_SRCDIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles + TEST_TARGET //:pass_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/bbb994006a201e70ea51a76491bba66br +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSKbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONSj +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR?bazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifestU +TEST_UNDECLARED_OUTPUTS_DIR6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsl + TEST_UNDECLARED_OUTPUTS_MANIFESTHbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFESTc +TEST_UNUSED_RUNFILES_LOG_FILEBbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_logT +TEST_WARNINGS_OUTPUT_FILE7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings +TEST_WORKSPACE_main +TZUTCE +XML_OUTPUT_FILE2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml $24"2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml2C"Abazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely29"7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings2M"Kbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs2D"Bbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_log2G"Ebazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failure2(%2J"Hbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFEST2M"Kbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONS2P"Nbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONS.pb: //:pass_testB +TestRunnerZdisk cache hit`hpx‚E +@eca04b5900fa4ed072925381966f72b38290fddecfc720ab6ea7fef50c0ad80d—ˆà§’" +À–±"À„=XÑ×`¢ ñú—ÖÀô­å +&2"*}'y +0bazel-out/k8-fastbuild/bin/sharded_test-0.paramsE +@bb9b5928db17b3f1215dc9e0405f5118783b5a7995020e6bc5fbd4df81f51d2e¥ +(2"&*'t)p +'bazel-out/k8-fastbuild/bin/sharded_testE +@0a8dce773490fe6720cadc64b758164f2f15a9b1a38c0ac818ada7a0bbb84e67�4±:® + /usr/bin/gcc +1@bazel-out/k8-fastbuild/bin/sharded_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 (2()://:sharded_testBCppLinkZdisk cache hit`hpx‚E +@6774a44f61fa2f7d823a17ba620338f16daa5a9958d0d086c47ef8ce1bfd6181’’ +À„=XË^`¢ ñú—Ö€û½Ø*{ +2bazel-out/k8-fastbuild/testlogs/pass_test/test.logE +@c4d97ed7504f0d4c742f57c5c7014ef33de0b8b077e90765a330ab231c2b09e5œ+2*#*,{ +2bazel-out/k8-fastbuild/testlogs/pass_test/test.xmlE +@e69b20191cc399c99510a46ad7322a20bd7fe10c2ac70a01c4e73bf5ad48969cýâ:ß +/external/bazel_tools/tools/test/generate-xml.sh +2bazel-out/k8-fastbuild/testlogs/pass_test/test.log +2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml +0 +0> + JAVA_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/bin@ +PYTHON_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles= + RUNFILES_DIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./pass_testi + TEST_INFRASTRUCTURE_FAILURE_FILEEbazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failurek +TEST_LOGSPLITTER_OUTPUT_FILEKbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs + TEST_NAME //:pass_test] +TEST_PREMATURE_EXIT_FILEAbazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely +TEST_SHARD_INDEX0 + TEST_SIZEmedium< + TEST_SRCDIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles + TEST_TARGET //:pass_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/bbb994006a201e70ea51a76491bba66b +TEST_TOTAL_SHARDS0r +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSKbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONSj +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR?bazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifestU +TEST_UNDECLARED_OUTPUTS_DIR6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsl + TEST_UNDECLARED_OUTPUTS_MANIFESTHbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFESTc +TEST_UNUSED_RUNFILES_LOG_FILEBbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_logT +TEST_WARNINGS_OUTPUT_FILE7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings +TEST_WORKSPACE_main +TZUTCE +XML_OUTPUT_FILE2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml +2(,: //:pass_testB +TestRunnerZdisk cache hit`hpx‚E +@b584e3038448fcd213780f7b60fdb56bb134d48a2ce96777cb5d5596f25a74ab’’ +€‰zXÌ/`¢ ñú—ÖÀ�ê„-2*)€.R| +0bazel-out/k8-fastbuild/bin/sharded_test.runfiles-2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026a /2*."# 02*."#�1Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855�2Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855L3"H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputsL4"H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs 52*."#Ÿ:œ +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely +TEST_SHARD_INDEX1^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/1615829df1fa3563387e50506a1f6692 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml /2D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely2(12I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure2(32Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@65823b7bbd88887a55104a2603f0cd27144b0b9a7f03b0cac4c813bde832c527—ˆà§’" +€’ô"À„=X˜Ü`¢ ñú—ÖÀº«úŸ:œ +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely +TEST_SHARD_INDEX0^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/baf5a5c95a10bc1fd7a56eb5c8e0a05a +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml 02D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely2(22I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure2(42Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@ae35bb8a01ebfe9e2a0aed70d708da493aa1d33f305661fa22e07f53e15c9e9e—ˆà§’" +€’ô"À„=X˜Ü`¢ ñú—ÖÀº«ú�6Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855L7"H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputsŸ:œ +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely +TEST_SHARD_INDEX2^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/c0c31cf43ef7d650f7ec6ed08aec4b02 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml 52D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely2(62I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure2(72Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@f814772f77f3e92e5cef55e6b0dc3b1db2d007d20513003199c7ee1612898823—ˆà§’" +À–±"À„=X˜Ü`¢ ñú—Ö€¿èú�8‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸ92*#8�:‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸ;2*#:�<‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xmlE +@da3be6ada8aa77a9ec8381222bb8358fcd7300c46bc8318e05b7cd792a471f60š�=‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xmlE +@8faa6c3e54a5d33b706bdd2917bf93f8e6522082b78cf016f23cc398558a7980š�>‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸ?2*#>ª:§ +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 1 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely +TEST_SHARD_INDEX0^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/baf5a5c95a10bc1fd7a56eb5c8e0a05a +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml 92(<://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@f170955a4da54e66b1a6040529fadb7c8bc18b0d15a87c7df4160a72bb039e5e’’ +€‰zX¨0`¢ ñú—րļ–�@‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xmlE +@cfe77d8780f07a6b93e2e76bb0806c9b5ec880e414605d03c0a9b92c7d0c57abš°:­ +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 2 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely +TEST_SHARD_INDEX1^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/1615829df1fa3563387e50506a1f6692 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml ;2(=://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@1829a1dbbc86771981413590c77863b3477fae7be62b1596a8f019422208491c’’ +€‰z"À„=X¨0`¢ ñú—րļ–ª:§ +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 3 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely +TEST_SHARD_INDEX2^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/c0c31cf43ef7d650f7ec6ed08aec4b02 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml ?2(@://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@104dc89ec01897d5979a03f7c0fce69f25f2f253c36c483719bb6ffdfdf26601’’ +€‰zX¨0`¢ ñú—ÖÀÈù–A2*~BRz +.bazel-out/k8-fastbuild/bin/flaky_test.runfilesA2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026a C2*B"#=D"9 +7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputsö:ó +-external/bazel_tools/tools/test/test-setup.sh + ./flaky_test? + JAVA_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binA +PYTHON_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles> + RUNFILES_DIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./flaky_testj + TEST_INFRASTRUCTURE_FAILURE_FILEFbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failurel +TEST_LOGSPLITTER_OUTPUT_FILELbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs^ +TEST_PREMATURE_EXIT_FILEBbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely + TEST_SIZEmedium= + TEST_SRCDIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles + TEST_TARGET //:flaky_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/d778484882c0bff6932e3335e6656a87s +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSLbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONSk +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR@bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifestV +TEST_UNDECLARED_OUTPUTS_DIR7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputsm + TEST_UNDECLARED_OUTPUTS_MANIFESTIbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFESTd +TEST_UNUSED_RUNFILES_LOG_FILECbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_logU +TEST_WARNINGS_OUTPUT_FILE8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings +TEST_WORKSPACE_main +TZUTCF +XML_OUTPUT_FILE3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml C25"3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml2D"Bbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely2:"8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings2N"Lbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs2E"Cbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_log2H"Fbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failure2(D2K"Ibazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFEST2N"Lbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONS2Q"Obazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONS.pb: //:flaky_testB +TestRunnerHR NON_ZERO_EXITZprocesswrapper-sandboxhpx‚E +@b95f7f7c6b2c699bbe8951bec716799645217fd1e03a20f1c86c40d731dc78aa—ˆà§’# +€¢•B€¢•¢ òú—Ö€‰•切 Ã€E| +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.logE +@f4f43b4414847462e830ccf2c6ed78b7f752d507751e9ac2f4edf797aee55ea0�F2*#E€G| +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xmlE +@c97b22d07a3c89431b8b3ecd40ca2e1ff7b17692616d90a4bf67f0606eae9cef²ü:ù +/external/bazel_tools/tools/test/generate-xml.sh +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.log +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml +0 +1? + JAVA_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binA +PYTHON_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles> + RUNFILES_DIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./flaky_testj + TEST_INFRASTRUCTURE_FAILURE_FILEFbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failurel +TEST_LOGSPLITTER_OUTPUT_FILELbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs + TEST_NAME //:flaky_test^ +TEST_PREMATURE_EXIT_FILEBbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely +TEST_SHARD_INDEX0 + TEST_SIZEmedium= + TEST_SRCDIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles + TEST_TARGET //:flaky_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/d778484882c0bff6932e3335e6656a87 +TEST_TOTAL_SHARDS0s +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSLbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONSk +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR@bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifestV +TEST_UNDECLARED_OUTPUTS_DIR7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputsm + TEST_UNDECLARED_OUTPUTS_MANIFESTIbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFESTd +TEST_UNUSED_RUNFILES_LOG_FILECbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_logU +TEST_WARNINGS_OUTPUT_FILE8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings +TEST_WORKSPACE_main +TZUTCF +XML_OUTPUT_FILE3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml F2(G: //:flaky_testB +TestRunnerZdisk cache hit`hpx‚E +@aec77e861fc71a6827175a35ca9248a9f51b1060f58b9f6debaa414a6c06f0b5’’ +€‰z"À„=XÎ/`¢ ñú—Ö€ÚÖý \ No newline at end of file diff --git a/tools/bazel/testdata/ci_analytics/regen.sh b/tools/bazel/testdata/ci_analytics/regen.sh new file mode 100755 index 000000000..7084c5893 --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/regen.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +# Regenerates real/*.json and real/*.binpb from a disposable scratch Bazel +# workspace. Run manually (not part of any test or CI job); the fixtures it +# produces are committed. This documents how they were made, so a reviewer +# can tell they are real Bazel 9 output and not hand-written JSON, and so +# they can be regenerated against a newer Bazel without re-deriving the +# scratch workspace from scratch. +# +# Isolation matters here as much as in bazel.yml itself, and more: the host +# that runs this script is a live operator workstation, not a disposable CI +# runner, so Bazel's own BEP `started` event and --client_env echo the whole +# ambient environment (hostname, user, SSH_CLIENT, auth tokens, gateway key +# paths, ...) straight into the fixture unless we scrub it. We do that in two +# layers: +# 1. Run Bazel under `env -i` with a minimal, scratch-local environment, so +# there is nothing secret in the environment to begin with. +# 2. Pass every raw BEP file through filter_bep() below, which keeps only +# the events and fields tools/bazel/ci_analytics_extract.py actually +# reads, before anything is copied into $out. This is defense in depth: +# even a future Bazel version that adds new ambient fields to `started` +# or a new event kind gets dropped by default, not included by default. +# +# --nohome_rc --nosystem_rc keep the home .bazelrc (which may carry a real +# --remote_executor) out of the invocation, and --output_user_root puts the +# repo-contents cache outside the scratch workspace. No remote flags are +# passed at all: these fixtures are deliberately local-only (disk_cache_hit, +# not remote_cache_hit). A synthetic remote-runner fixture is built in +# ci_analytics_extract_test.py instead, from its own varint encoder, since +# this repo's sandbox has no RBE endpoint to run against. +set -euo pipefail + +real_bazel=$(command -v bazel) + +scratch=$(mktemp -d) +userroot=$(mktemp -d) +diskcache=$(mktemp -d) +home="$scratch/home" +mkdir -p "$home" +trap 'rm -rf "$scratch" "$userroot" "$diskcache"' EXIT + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +cp "$repo_root/.bazelversion" "$scratch/.bazelversion" + +cat >"$scratch/MODULE.bazel" <<'EOF' +module(name = "s2scratch", version = "0.0.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +EOF + +cat >"$scratch/BUILD.bazel" <<'EOF' +load("@rules_cc//cc:defs.bzl", "cc_test") + +cc_test( + name = "pass_test", + srcs = ["main_pass.c"], +) + +cc_test( + name = "sharded_test", + srcs = ["main_shard.c"], + shard_count = 3, +) + +cc_test( + name = "flaky_test", + srcs = ["main_fail.c"], +) + +genrule( + name = "gen_out", + outs = ["gen_out.txt"], + cmd = "echo hello > $@", +) +EOF + +cat >"$scratch/main_pass.c" <<'EOF' +int main(void) { return 0; } +EOF + +cat >"$scratch/main_fail.c" <<'EOF' +int main(void) { return 1; } +EOF + +cat >"$scratch/main_shard.c" <<'EOF' +#include +#include + +int main(void) { + const char *status_file = getenv("TEST_SHARD_STATUS_FILE"); + if (status_file != NULL) { + FILE *f = fopen(status_file, "w"); + if (f != NULL) { + fclose(f); + } + } + return 0; +} +EOF + +cd "$scratch" + +# A fully scrubbed environment: no inherited secrets, SSH state, gateway +# addresses or tokens. PATH is just enough for a C toolchain and bazel's own +# exec needs; HOME is scratch-local so nothing in it (no ~/.bazelrc either) +# can leak in, and so Bazel's own "user"/"host" fields come from this +# sandboxed identity rather than the operator's. +run_bazel() { + env -i \ + HOME="$home" \ + PATH=/usr/bin:/bin \ + USER=fixture \ + LOGNAME=fixture \ + TMPDIR=/tmp \ + "$real_bazel" --nohome_rc --nosystem_rc "--output_user_root=$userroot" "$@" +} + +out="$1" # destination dir, e.g. tools/bazel/testdata/ci_analytics/real +mkdir -p "$out" + +# filter_bep keeps only the BEP events and fields +# tools/bazel/ci_analytics_extract.py reads (plus buildToolLogs' "name" and +# "contents", with "uri" stripped, since those are local filesystem paths and +# not needed to recompute anything the extractor reports). Everything else — +# unstructuredCommandLine, structuredCommandLine, optionsParsed, +# workspaceStatus, buildMetadata, progress, and the leaky parts of started — +# is dropped entirely rather than merely left unused, so the fixture itself +# carries nothing a reviewer reading the raw JSON could call a leak. +filter_bep() { + python3 - "$1" "$2" <<'PY' +import json +import sys + +src, dst = sys.argv[1], sys.argv[2] + +# Positive allowlists only, one per event kind: every key below is one +# ci_analytics_extract.py actually reads (cross-checked against +# build_timing/build_tests/build_action_data). Nothing else survives, +# including keys a future Bazel version might add -- a denylist (drop just +# the known-leaky keys) was tried first and missed testSummary.passed/ +# failed[].uri (a file:// local path) entirely; an allowlist +# can't miss a key like that by construction, it would just never be kept. +STARTED_KEEP = {"uuid", "command", "buildToolVersion", "startTimeMillis"} +TESTRESULT_KEEP = {"status", "cachedLocally", "testAttemptDuration", "testAttemptDurationMillis"} +EXECINFO_KEEP = {"strategy", "timingBreakdown"} +TIMING_CHILD_KEEP = {"name", "time"} +TESTSUMMARY_KEEP = {"overallStatus", "shardCount", "attemptCount", "totalNumCached"} +FINISHED_KEEP = {"finishTimeMillis"} +EXITCODE_KEEP = {"code", "name"} +LOG_ENTRY_KEEP = {"name", "contents"} + + +def pick(d, keys): + return {k: v for k, v in d.items() if k in keys} if isinstance(d, dict) else {} + + +def filter_test_result(tr): + if not isinstance(tr, dict): + return {} + out = pick(tr, TESTRESULT_KEEP) + ei = tr.get("executionInfo") + if isinstance(ei, dict): + filtered_ei = pick(ei, EXECINFO_KEEP) + tbd = ei.get("timingBreakdown") + if isinstance(tbd, dict) and isinstance(tbd.get("child"), list): + filtered_ei["timingBreakdown"] = { + "child": [pick(c, TIMING_CHILD_KEEP) for c in tbd["child"] if isinstance(c, dict)] + } + out["executionInfo"] = filtered_ei + return out + + +def filter_build_metrics(bm): + if not isinstance(bm, dict): + return {} + out = {} + tm = bm.get("timingMetrics") + if isinstance(tm, dict): + out["timingMetrics"] = pick( + tm, + { + "cpuTimeInMs", + "wallTimeInMs", + "analysisPhaseTimeInMs", + "executionPhaseTimeInMs", + "actionsExecutionStartInMs", + "criticalPathTime", + "criticalPathTimeInMs", + }, + ) + asum = bm.get("actionSummary") + if isinstance(asum, dict): + filtered_asum = pick(asum, {"actionsCreated", "actionsExecuted"}) + if isinstance(asum.get("runnerCount"), list): + filtered_asum["runnerCount"] = [ + pick(rc, {"name", "count", "execKind"}) for rc in asum["runnerCount"] if isinstance(rc, dict) + ] + acs = asum.get("actionCacheStatistics") + if isinstance(acs, dict): + filtered_asum["actionCacheStatistics"] = pick(acs, {"hits", "misses"}) + if isinstance(asum.get("actionData"), list): + filtered_asum["actionData"] = [ + pick(ad, {"mnemonic", "actionsExecuted", "actionsCreated", "userTime", "systemTime"}) + for ad in asum["actionData"] + if isinstance(ad, dict) + ] + out["actionSummary"] = filtered_asum + pm = bm.get("packageMetrics") + if isinstance(pm, dict): + out["packageMetrics"] = pick(pm, {"packagesLoaded"}) + tgm = bm.get("targetMetrics") + if isinstance(tgm, dict): + out["targetMetrics"] = pick(tgm, {"targetsConfigured"}) + nmw = bm.get("networkMetrics") + nm = nmw.get("systemNetworkStats") if isinstance(nmw, dict) else None + if isinstance(nm, dict): + out["networkMetrics"] = {"systemNetworkStats": pick(nm, {"bytesRecv", "bytesSent"})} + return out + + +with open(src) as fin, open(dst, "w") as fout: + for line in fin: + line = line.strip() + if not line: + continue + event = json.loads(line) + kept = {} + # "id" carries the BEP identifier for this event (e.g. + # id.testResult.label / id.testSummary.label), which + # ci_analytics_extract.py reads to group test rows by label. It is + # not leaky on its own (a label, run/shard/attempt numbers), so it + # is kept verbatim alongside whichever event body below is kept. + if "id" in event: + kept["id"] = event["id"] + if "started" in event and isinstance(event["started"], dict): + kept["started"] = pick(event["started"], STARTED_KEEP) + elif "testResult" in event: + kept["testResult"] = filter_test_result(event["testResult"]) + elif "testSummary" in event and isinstance(event["testSummary"], dict): + # passed/failed (TestFile{uri, ...} lists) are never read by + # the extractor at all: dropped entirely, not merely + # uri-stripped, along with anything else not in the allowlist. + kept["testSummary"] = pick(event["testSummary"], TESTSUMMARY_KEEP) + elif "buildMetrics" in event: + kept["buildMetrics"] = filter_build_metrics(event["buildMetrics"]) + elif "finished" in event and isinstance(event["finished"], dict): + fin_ev = pick(event["finished"], FINISHED_KEEP) + ec = event["finished"].get("exitCode") + if isinstance(ec, dict): + fin_ev["exitCode"] = pick(ec, EXITCODE_KEEP) + kept["finished"] = fin_ev + elif "buildToolLogs" in event: + btl = event["buildToolLogs"] + logs = btl.get("log") if isinstance(btl, dict) else None + kept["buildToolLogs"] = { + "log": [pick(entry, LOG_ENTRY_KEEP) for entry in (logs or []) if isinstance(entry, dict)] + } + else: + kept.pop("id", None) + continue + fout.write(json.dumps(kept) + "\n") +PY +} + +# trim_profile keeps only the "Fetching repository" category entries the +# extractor's iter_repo_fetch() reads, dropping the (much larger, and +# otherwise unredacted) rest of the Chrome trace. Written one event per +# line -- {"traceEvents":[\n{...},\n{...}\n]} -- matching the real, +# streamable shape Bazel itself writes a JSON --profile in (as opposed to +# json.dump()'s single line): iter_repo_fetch() reads the profile line by +# line specifically so it never has to hold the whole (potentially huge) +# trace in memory, and a single-line file defeats that path entirely +# (every "Fetching repository" entry, if there's more than one, ends up on +# the one line containing the literal "traceEvents" key instead, which +# doesn't have a "cat" of its own and is skipped -- repo_fetch would always +# be empty, and the test for it would pass vacuously). +trim_profile() { + python3 - "$1" "$2" <<'PY' +import json +import sys + +src, dst = sys.argv[1], sys.argv[2] +with open(src) as f: + doc = json.load(f) +events = [ + e for e in doc.get("traceEvents", []) if e.get("cat") == "Fetching repository" +] +with open(dst, "w") as f: + f.write('{"traceEvents":[\n') + f.write(",\n".join(json.dumps(e) for e in events)) + f.write("\n]}\n") +PY +} + +# Cold: no disk cache. pass_test and sharded_test pass; flaky_test fails +# (deliberately, so the fixture has a FAILED testResult too). +run_bazel test //... \ + --keep_going \ + --build_event_json_file="$scratch/cold-bep.raw.json" \ + --execution_log_compact_file="$scratch/cold-exec.log" \ + --profile="$scratch/cold-profile.raw.json" || true +filter_bep "$scratch/cold-bep.raw.json" "$out/cold-bep.json" +zstd -dc --long=0 "$scratch/cold-exec.log" >"$out/cold-exec.binpb" +trim_profile "$scratch/cold-profile.raw.json" "$out/cold-profile.json" + +# Warm: first run with --disk_cache set populates it (its own actions are +# still local execs; the disk cache is empty going in). Not copied to $out +# on its own; it only sets up the disk cache for the post-clean run below. +run_bazel test //... \ + --keep_going \ + "--disk_cache=$diskcache" \ + --build_event_json_file="$scratch/warm1-bep.json" \ + --execution_log_compact_file="$scratch/warm1-exec.log" || true + +# Post-clean, with the disk cache warm1 just populated: every action is a +# disk_cache_hit except flaky_test's TestRunner spawn (Bazel never caches a +# failed test's result, so it reruns and misses). +run_bazel clean +run_bazel test //... \ + --keep_going \ + "--disk_cache=$diskcache" \ + --build_event_json_file="$scratch/postclean-bep.raw.json" \ + --execution_log_compact_file="$scratch/postclean-exec.log" || true +filter_bep "$scratch/postclean-bep.raw.json" "$out/postclean-bep.json" +zstd -dc --long=0 "$scratch/postclean-exec.log" >"$out/postclean-exec.binpb" + +echo "wrote fixtures to $out" >&2