* chore(migration-test): extend the release catalog through v1.3.1 The catalog stopped at v1.1.2 (the v1.2 upgrade census). The migration corpus already upgrades from v1.2.2, and the Bazel targets that replace migration-test.yml and cross-version-smoke.yml take their pinned release binaries from the catalog (@bd_releases), so it has to cover the releases since. Regenerated with `catalog generate` after raising maximumVersion to v1.3.1: five new stable versions (v1.2.0, v1.2.1, v1.2.2, v1.3.0, v1.3.1; three with a linux/amd64 asset) and five new proxy prereleases; no existing entry, drift record or repository-only exclusion changed. The count constants and the identity digest are updated to match, and the README now says how to add a release. Refs: ga-96smfk.19 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * ci(bazel): run the historical-upgrade corpus under Bazel; retire migration-test.yml migration-test.yml ran scripts/migration-test/run.sh --version <release> for 14 reviewed releases in 3 shards on GitHub runners: it downloaded each release archive, built v0.9.1 from source with Go 1.25 through the module proxy, installed libicu74 and the dolt 2.1.8 runtime, all over the network. //tests/migration now has one target per corpus release (historical_upgrade_<release>_test) plus legacy_bridge_test, all hermetic: - Published releases come from @bd_releases (the release catalog's sha256-verified assets) through a per-target directory (HISTORICAL_RELEASE_DIR); lib/binary.sh uses it instead of downloading and still checks each binary's self-reported version. - v0.9.1 is //tests/migration:bd_source_tag, the same `go mod download` + `go build -trimpath` with CGO and Go 1.25.0, run offline as a build action: the module files come from tests/migration/source_tag_modules.json through a file:// GOPROXY tree (tools/bazel/go_module_proxy.bzl), the release's own zip is the catalog's source_zip, and go checks every dependency against the release's go.sum (SOURCE_TAG_SQLITE_BIN). - The external Dolt runtime is @dolt_test_runtime_linux_amd64 (dolt.bzl's dolt_release gains a version attribute; 2.1.8 is pinned to versions.sh's digest), passed as DOLT_BIN to the server-Dolt and wisp-plane releases. - The workers already provide jq, python3, sqlite3 and libicu74. The targets are untagged, so bazel-test runs them remotely on every PR, merge group and push; result caching replaces the path filter. Every release also passed inside a loopback-only network namespace. scripts/migration_corpus_bazel_test.go keeps the BUILD lists equal to versions.sh's corpus, the catalog digests equal to STRICT_RELEASE_SHA256, the source-tag lock equal to the catalog's source_zip, and the Dolt runtime pin equal to versions.sh's. migration-test.yml and the policy tests that pinned its shards, caches, apt step and runner are removed. Refs: ga-96smfk.19 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * ci(bazel): run the historical-upgrade targets in the PR-core lane only The integration lane runs every untagged target too, and its check_testcases.py requires Go testcases, which a shell test's test.xml does not list. A new pr-core-only tag keeps //tests/migration out of --config=integration (bazelIntegrationExcludedTags adds it), so the corpus runs once, in bazel-test, against the release-shaped bd_for_tests (no integration tag). build_source_tag.sh carries the build-tags opt-out: it builds a historical release with that release's own go.mod, which predates gms_pure_go. Refs: ga-96smfk.19 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
154 lines
5.6 KiB
Go
154 lines
5.6 KiB
Go
package scripts_test
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// aptStepBudget describes one bounded `apt-get` step: how long its own inner
|
|
// `timeout` calls may run, and the step-level `timeout-minutes` that backstops
|
|
// them.
|
|
//
|
|
// The components are declared here rather than only pinning the final minute
|
|
// count so that the two halves are checked against each other. A cap that
|
|
// lands below the step's own worst case reproduces exactly the undiagnosable
|
|
// mid-flight kill this hardening exists to remove, just at a smaller number --
|
|
// and that is invisible if the test only asserts the cap's literal value.
|
|
type aptStepBudget struct {
|
|
workflow string
|
|
job string
|
|
step string
|
|
|
|
updateAttempts int // iterations of the `for attempt in ...` loop
|
|
updateTimeout int // seconds allowed per `apt-get update` attempt
|
|
backoff int // seconds slept between attempts (never after the last)
|
|
installTimeout int // seconds allowed for `apt-get install`
|
|
installPkgs string
|
|
|
|
capMinutes int // the step's `timeout-minutes`
|
|
}
|
|
|
|
// worstCaseSeconds is the longest the step's own bounds permit it to run.
|
|
func (b aptStepBudget) worstCaseSeconds() int {
|
|
return b.updateAttempts*b.updateTimeout +
|
|
(b.updateAttempts-1)*b.backoff +
|
|
b.installTimeout
|
|
}
|
|
|
|
// loopHeader is the `for` header the declared attempt count implies.
|
|
func (b aptStepBudget) loopHeader() string {
|
|
attempts := make([]string, 0, b.updateAttempts)
|
|
for i := 1; i <= b.updateAttempts; i++ {
|
|
attempts = append(attempts, strconv.Itoa(i))
|
|
}
|
|
return "for attempt in " + strings.Join(attempts, " ")
|
|
}
|
|
|
|
func aptStepBudgets() []aptStepBudget {
|
|
return []aptStepBudget{
|
|
{
|
|
workflow: "release.yml",
|
|
job: "goreleaser",
|
|
step: "Install cross-compilation toolchains and signing tools",
|
|
updateAttempts: 3,
|
|
updateTimeout: 120,
|
|
backoff: 10,
|
|
installTimeout: 300,
|
|
installPkgs: "gcc-mingw-w64-x86-64 gcc-aarch64-linux-gnu osslsigncode",
|
|
capMinutes: 13,
|
|
},
|
|
{
|
|
workflow: "pr.yml",
|
|
job: "windows-test-binaries",
|
|
step: "Install mingw cross-compiler",
|
|
updateAttempts: 3,
|
|
updateTimeout: 120,
|
|
backoff: 10,
|
|
installTimeout: 300,
|
|
installPkgs: "gcc-mingw-w64-x86-64",
|
|
capMinutes: 12,
|
|
},
|
|
{
|
|
workflow: "main.yml",
|
|
job: "windows-test-binaries-cache",
|
|
step: "Install mingw cross-compiler",
|
|
updateAttempts: 3,
|
|
updateTimeout: 120,
|
|
backoff: 10,
|
|
installTimeout: 300,
|
|
installPkgs: "gcc-mingw-w64-x86-64",
|
|
capMinutes: 12,
|
|
},
|
|
}
|
|
}
|
|
|
|
// TestAptGetStepsAreBoundedAndRetried pins the bounded/retried invocations, so
|
|
// a revert to a bare `apt-get update` -- the unbounded shape that burned whole
|
|
// job budgets on a stalled mirror -- goes red.
|
|
func TestAptGetStepsAreBoundedAndRetried(t *testing.T) {
|
|
for _, b := range aptStepBudgets() {
|
|
t.Run(b.workflow+"/"+b.step, func(t *testing.T) {
|
|
step := readCIWorkflow(t, b.workflow).job(t, b.job).step(t, b.step)
|
|
|
|
for _, required := range []string{
|
|
fmt.Sprintf("sudo timeout %d apt-get", b.updateTimeout),
|
|
"-o Acquire::Retries=3",
|
|
"-o Acquire::http::Timeout=30",
|
|
"-o Acquire::https::Timeout=30",
|
|
b.loopHeader(),
|
|
// Backoff between attempts, but not after the final one.
|
|
fmt.Sprintf("if [ \"$attempt\" -lt %d ]; then", b.updateAttempts),
|
|
fmt.Sprintf("sleep %d", b.backoff),
|
|
// Exhausting the retries proceeds best-effort against the
|
|
// cached index; the warning is what keeps that visible in the
|
|
// job log instead of a silent fall-through.
|
|
fmt.Sprintf("echo \"::warning::apt-get update failed after %d attempts; proceeding with cached index\"", b.updateAttempts),
|
|
fmt.Sprintf("sudo timeout %d apt-get install -y %s", b.installTimeout, b.installPkgs),
|
|
} {
|
|
if !strings.Contains(step.Run, required) {
|
|
t.Errorf("%s command does not contain %q:\n%s", b.step, required, step.Run)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestAptGetStepTimeoutsExceedTheirOwnBudget is the relation the first round of
|
|
// this hardening got backwards: the step-level cap was set *tighter* than the
|
|
// inner `timeout` calls it was meant to backstop.
|
|
func TestAptGetStepTimeoutsExceedTheirOwnBudget(t *testing.T) {
|
|
for _, b := range aptStepBudgets() {
|
|
t.Run(b.workflow+"/"+b.step, func(t *testing.T) {
|
|
workflow := readCIWorkflow(t, b.workflow)
|
|
job := workflow.job(t, b.job)
|
|
step := job.step(t, b.step)
|
|
|
|
if step.TimeoutMinutes != b.capMinutes {
|
|
t.Errorf("%s timeout-minutes = %d, want %d", b.step, step.TimeoutMinutes, b.capMinutes)
|
|
}
|
|
|
|
// Read the relations off the workflow itself, never off capMinutes:
|
|
// checking a declared constant against the budget it was derived
|
|
// from is self-satisfying, and would stay green while the YAML
|
|
// drifted underneath it.
|
|
capSeconds := step.TimeoutMinutes * 60
|
|
if capSeconds <= b.worstCaseSeconds() {
|
|
t.Errorf("%s timeout-minutes = %d (%ds) does not exceed its own worst case of %ds "+
|
|
"(%d update attempts x %ds + %d backoffs x %ds + %ds install); "+
|
|
"raise the cap or shrink the inner budgets",
|
|
b.step, step.TimeoutMinutes, capSeconds, b.worstCaseSeconds(),
|
|
b.updateAttempts, b.updateTimeout, b.updateAttempts-1, b.backoff, b.installTimeout)
|
|
}
|
|
|
|
// Where the job carries its own cap, the step must fail first --
|
|
// otherwise the whole job is cancelled and the log gives no hint
|
|
// that apt was the thing that stalled.
|
|
if job.TimeoutMinutes > 0 && step.TimeoutMinutes >= job.TimeoutMinutes {
|
|
t.Errorf("%s timeout-minutes = %d is not below job %q timeout-minutes = %d",
|
|
b.step, step.TimeoutMinutes, b.job, job.TimeoutMinutes)
|
|
}
|
|
})
|
|
}
|
|
}
|