beads/scripts/ci_workflow_apt_get_timeout_test.go
Jim Wordelman 6ec78f3a2d
fix: bound and retry apt-get calls in CI to avoid unbounded 15-minute job timeout (#5810)
* test(feat): red — Fix: unbounded apt-get in CI burns the whole 15m job timeout (migration-test.yml, release.yml) (refs be-na2q)

* fix(feat): green — Fix: unbounded apt-get in CI burns the whole 15m job timeout (migration-test.yml, release.yml) (refs be-na2q)

* fix(ci): make apt-get update retry exhaustion deliberate, not a silent fall-through

After three failed `apt-get update` attempts the loop simply ended and the
step went on to `apt-get install` against a stale index, with nothing in the
log saying the index was never refreshed. That contradicted the PR body's
claim that the step "fails fast with a clear error".

Proceeding best-effort is the right behavior -- the runner image's cached
index installs these packages far more often than not, and failing here would
turn a transient mirror outage into a red build -- so make it an explicit,
logged decision instead of a fall-through:

  - track whether any attempt succeeded and emit
    `:⚠️:apt-get update failed after 3 attempts; proceeding with
    cached index` when none did
  - drop "; retrying" from the per-attempt warning, which was wrong on the
    final attempt
  - skip the 10s backoff after the last attempt, where there is nothing left
    to wait for

The workflow tests pin both new strings, so a revert to the silent
fall-through goes red.

* fix(ci): raise apt step timeout-minutes above the retry budget they backstop

The step-level caps were tighter than the inner `timeout` calls they were
meant to backstop, so they reproduced the mid-flight kill this hardening
exists to remove -- just at 6 and 8 minutes instead of 15:

  migration-test.yml  worst case 3x120s + 2x10s + 180s = 560s vs cap 360s
  release.yml         worst case 3x120s + 2x10s + 300s = 680s vs cap 480s

Raise them to 11m and 13m, each ~100s above its own worst case, and record
the arithmetic next to the value so the next edit re-derives it. 11m also
stays under the historical-upgrades job's own 15m budget, so a stuck step
fails as a step rather than cancelling the job with no apt-specific signal.
The goreleaser job sets no job-level timeout-minutes at all, which makes its
step cap the only bound on a stuck apt there.

The workflow tests now assert the relations instead of only pinning the
minute counts:

  - step timeout-minutes must exceed the step's own worst-case budget,
    computed from the attempt count, per-attempt timeout, backoff and
    install timeout that the same table pins against the YAML
  - it must stay below the job's timeout-minutes where the job sets one

Both read the value off the workflow rather than off the table's own
constant, so they stay honest if the YAML drifts.
2026-08-19 11:47:22 -07:00

143 lines
5.3 KiB
Go

package scripts_test
import (
"fmt"
"strconv"
"strings"
"testing"
)
// aptStepBudget describes one bounded `apt-get` step: how long its own inner
// `timeout` calls may run, and the step-level `timeout-minutes` that backstops
// them.
//
// The components are declared here rather than only pinning the final minute
// count so that the two halves are checked against each other. A cap that
// lands below the step's own worst case reproduces exactly the undiagnosable
// mid-flight kill this hardening exists to remove, just at a smaller number --
// and that is invisible if the test only asserts the cap's literal value.
type aptStepBudget struct {
workflow string
job string
step string
updateAttempts int // iterations of the `for attempt in ...` loop
updateTimeout int // seconds allowed per `apt-get update` attempt
backoff int // seconds slept between attempts (never after the last)
installTimeout int // seconds allowed for `apt-get install`
installPkgs string
capMinutes int // the step's `timeout-minutes`
}
// worstCaseSeconds is the longest the step's own bounds permit it to run.
func (b aptStepBudget) worstCaseSeconds() int {
return b.updateAttempts*b.updateTimeout +
(b.updateAttempts-1)*b.backoff +
b.installTimeout
}
// loopHeader is the `for` header the declared attempt count implies.
func (b aptStepBudget) loopHeader() string {
attempts := make([]string, 0, b.updateAttempts)
for i := 1; i <= b.updateAttempts; i++ {
attempts = append(attempts, strconv.Itoa(i))
}
return "for attempt in " + strings.Join(attempts, " ")
}
func aptStepBudgets() []aptStepBudget {
return []aptStepBudget{
{
workflow: "migration-test.yml",
job: "historical-upgrades",
step: "Install test dependencies",
updateAttempts: 3,
updateTimeout: 120,
backoff: 10,
installTimeout: 180,
installPkgs: "jq libicu74",
capMinutes: 11,
},
{
workflow: "release.yml",
job: "goreleaser",
step: "Install cross-compilation toolchains and signing tools",
updateAttempts: 3,
updateTimeout: 120,
backoff: 10,
installTimeout: 300,
installPkgs: "gcc-mingw-w64-x86-64 gcc-aarch64-linux-gnu osslsigncode",
capMinutes: 13,
},
}
}
// TestAptGetStepsAreBoundedAndRetried pins the bounded/retried invocations, so
// a revert to a bare `apt-get update` -- the unbounded shape that burned whole
// job budgets on a stalled mirror -- goes red.
func TestAptGetStepsAreBoundedAndRetried(t *testing.T) {
for _, b := range aptStepBudgets() {
t.Run(b.workflow+"/"+b.step, func(t *testing.T) {
step := readCIWorkflow(t, b.workflow).job(t, b.job).step(t, b.step)
for _, required := range []string{
fmt.Sprintf("sudo timeout %d apt-get", b.updateTimeout),
"-o Acquire::Retries=3",
"-o Acquire::http::Timeout=30",
"-o Acquire::https::Timeout=30",
b.loopHeader(),
// Backoff between attempts, but not after the final one.
fmt.Sprintf("if [ \"$attempt\" -lt %d ]; then", b.updateAttempts),
fmt.Sprintf("sleep %d", b.backoff),
// Exhausting the retries proceeds best-effort against the
// cached index; the warning is what keeps that visible in the
// job log instead of a silent fall-through.
fmt.Sprintf("echo \"::warning::apt-get update failed after %d attempts; proceeding with cached index\"", b.updateAttempts),
fmt.Sprintf("sudo timeout %d apt-get install -y %s", b.installTimeout, b.installPkgs),
} {
if !strings.Contains(step.Run, required) {
t.Errorf("%s command does not contain %q:\n%s", b.step, required, step.Run)
}
}
})
}
}
// TestAptGetStepTimeoutsExceedTheirOwnBudget is the relation the first round of
// this hardening got backwards: the step-level cap was set *tighter* than the
// inner `timeout` calls it was meant to backstop.
func TestAptGetStepTimeoutsExceedTheirOwnBudget(t *testing.T) {
for _, b := range aptStepBudgets() {
t.Run(b.workflow+"/"+b.step, func(t *testing.T) {
workflow := readCIWorkflow(t, b.workflow)
job := workflow.job(t, b.job)
step := job.step(t, b.step)
if step.TimeoutMinutes != b.capMinutes {
t.Errorf("%s timeout-minutes = %d, want %d", b.step, step.TimeoutMinutes, b.capMinutes)
}
// Read the relations off the workflow itself, never off capMinutes:
// checking a declared constant against the budget it was derived
// from is self-satisfying, and would stay green while the YAML
// drifted underneath it.
capSeconds := step.TimeoutMinutes * 60
if capSeconds <= b.worstCaseSeconds() {
t.Errorf("%s timeout-minutes = %d (%ds) does not exceed its own worst case of %ds "+
"(%d update attempts x %ds + %d backoffs x %ds + %ds install); "+
"raise the cap or shrink the inner budgets",
b.step, step.TimeoutMinutes, capSeconds, b.worstCaseSeconds(),
b.updateAttempts, b.updateTimeout, b.updateAttempts-1, b.backoff, b.installTimeout)
}
// Where the job carries its own cap, the step must fail first --
// otherwise the whole job is cancelled and the log gives no hint
// that apt was the thing that stalled.
if job.TimeoutMinutes > 0 && step.TimeoutMinutes >= job.TimeoutMinutes {
t.Errorf("%s timeout-minutes = %d is not below job %q timeout-minutes = %d",
b.step, step.TimeoutMinutes, b.job, job.TimeoutMinutes)
}
})
}
}