beads/scripts/ci_workflow_apt_get_timeout_test.go
Julian Knutsen 84390b03ab
ci(bazel): run every Windows/macOS job on Blacksmith; retire the Linux preflight leg and the cross-compiled Windows pair (#7355)
- pr.yml's Windows jobs and the macOS/Windows legs of pr-preflight-platforms
  and check-doc-freshness-platforms run on Blacksmith for every PR and merge
  group, forks and Dependabot included: no windows-latest/macos-latest
  fallback (Blacksmith serves this org's fork PRs). Matrix legs name their
  label (runs-on: ${{ matrix.runner }}); the same-repo "matrix marker"
  ternary and its policy plumbing are gone.
- pr-preflight-platforms drops its Linux leg: its tests run under Bazel on
  rbe-west (//scripts:scripts_test, //cmd/bd:bd_test,
  //internal/config:config_test).
- The Linux-cross-compiled Windows path is retired: windows-test-binaries,
  its two -prebuilt twins, main.yml's windows-test-binaries-cache, the
  WINDOWS_PREBUILT_REQUIRED flag and the build script/manifest. The native
  pair (test-windows-liveness, worktree-remove-windows) is the required run.
- main.yml's test-windows and blacksmith-macos-go-build-cache drop their
  GitHub-hosted venue legs (no GitHub-hosted consumer left).

Gate names unchanged. ga-96smfk.22

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-07 10:41:14 +00:00

132 lines
5 KiB
Go

package scripts_test
import (
"fmt"
"strconv"
"strings"
"testing"
)
// aptStepBudget describes one bounded `apt-get` step: how long its own inner
// `timeout` calls may run, and the step-level `timeout-minutes` that backstops
// them.
//
// The components are declared here rather than only pinning the final minute
// count so that the two halves are checked against each other. A cap that
// lands below the step's own worst case reproduces exactly the undiagnosable
// mid-flight kill this hardening exists to remove, just at a smaller number --
// and that is invisible if the test only asserts the cap's literal value.
type aptStepBudget struct {
workflow string
job string
step string
updateAttempts int // iterations of the `for attempt in ...` loop
updateTimeout int // seconds allowed per `apt-get update` attempt
backoff int // seconds slept between attempts (never after the last)
installTimeout int // seconds allowed for `apt-get install`
installPkgs string
capMinutes int // the step's `timeout-minutes`
}
// worstCaseSeconds is the longest the step's own bounds permit it to run.
func (b aptStepBudget) worstCaseSeconds() int {
return b.updateAttempts*b.updateTimeout +
(b.updateAttempts-1)*b.backoff +
b.installTimeout
}
// loopHeader is the `for` header the declared attempt count implies.
func (b aptStepBudget) loopHeader() string {
attempts := make([]string, 0, b.updateAttempts)
for i := 1; i <= b.updateAttempts; i++ {
attempts = append(attempts, strconv.Itoa(i))
}
return "for attempt in " + strings.Join(attempts, " ")
}
func aptStepBudgets() []aptStepBudget {
return []aptStepBudget{
{
workflow: "release.yml",
job: "goreleaser",
step: "Install cross-compilation toolchains and signing tools",
updateAttempts: 3,
updateTimeout: 120,
backoff: 10,
installTimeout: 300,
installPkgs: "gcc-mingw-w64-x86-64 gcc-aarch64-linux-gnu osslsigncode",
capMinutes: 13,
},
}
}
// TestAptGetStepsAreBoundedAndRetried pins the bounded/retried invocations, so
// a revert to a bare `apt-get update` -- the unbounded shape that burned whole
// job budgets on a stalled mirror -- goes red.
func TestAptGetStepsAreBoundedAndRetried(t *testing.T) {
for _, b := range aptStepBudgets() {
t.Run(b.workflow+"/"+b.step, func(t *testing.T) {
step := readCIWorkflow(t, b.workflow).job(t, b.job).step(t, b.step)
for _, required := range []string{
fmt.Sprintf("sudo timeout %d apt-get", b.updateTimeout),
"-o Acquire::Retries=3",
"-o Acquire::http::Timeout=30",
"-o Acquire::https::Timeout=30",
b.loopHeader(),
// Backoff between attempts, but not after the final one.
fmt.Sprintf("if [ \"$attempt\" -lt %d ]; then", b.updateAttempts),
fmt.Sprintf("sleep %d", b.backoff),
// Exhausting the retries proceeds best-effort against the
// cached index; the warning is what keeps that visible in the
// job log instead of a silent fall-through.
fmt.Sprintf("echo \"::warning::apt-get update failed after %d attempts; proceeding with cached index\"", b.updateAttempts),
fmt.Sprintf("sudo timeout %d apt-get install -y %s", b.installTimeout, b.installPkgs),
} {
if !strings.Contains(step.Run, required) {
t.Errorf("%s command does not contain %q:\n%s", b.step, required, step.Run)
}
}
})
}
}
// TestAptGetStepTimeoutsExceedTheirOwnBudget is the relation the first round of
// this hardening got backwards: the step-level cap was set *tighter* than the
// inner `timeout` calls it was meant to backstop.
func TestAptGetStepTimeoutsExceedTheirOwnBudget(t *testing.T) {
for _, b := range aptStepBudgets() {
t.Run(b.workflow+"/"+b.step, func(t *testing.T) {
workflow := readCIWorkflow(t, b.workflow)
job := workflow.job(t, b.job)
step := job.step(t, b.step)
if step.TimeoutMinutes != b.capMinutes {
t.Errorf("%s timeout-minutes = %d, want %d", b.step, step.TimeoutMinutes, b.capMinutes)
}
// Read the relations off the workflow itself, never off capMinutes:
// checking a declared constant against the budget it was derived
// from is self-satisfying, and would stay green while the YAML
// drifted underneath it.
capSeconds := step.TimeoutMinutes * 60
if capSeconds <= b.worstCaseSeconds() {
t.Errorf("%s timeout-minutes = %d (%ds) does not exceed its own worst case of %ds "+
"(%d update attempts x %ds + %d backoffs x %ds + %ds install); "+
"raise the cap or shrink the inner budgets",
b.step, step.TimeoutMinutes, capSeconds, b.worstCaseSeconds(),
b.updateAttempts, b.updateTimeout, b.updateAttempts-1, b.backoff, b.installTimeout)
}
// Where the job carries its own cap, the step must fail first --
// otherwise the whole job is cancelled and the log gives no hint
// that apt was the thing that stalled.
if job.TimeoutMinutes > 0 && step.TimeoutMinutes >= job.TimeoutMinutes {
t.Errorf("%s timeout-minutes = %d is not below job %q timeout-minutes = %d",
b.step, step.TimeoutMinutes, b.job, job.TimeoutMinutes)
}
})
}
}