diff --git a/.github/workflows/cross-version-smoke.yml b/.github/workflows/cross-version-smoke.yml deleted file mode 100644 index 938410be8..000000000 --- a/.github/workflows/cross-version-smoke.yml +++ /dev/null @@ -1,175 +0,0 @@ -name: Cross-Version Smoke Tests - -on: - # On tag push (release gate): test the last 30 releases for broad regression coverage - push: - tags: - - 'v*' - - # On PR to main or a release branch: test only the last 5 releases for fast - # feedback. Release-targeted PRs are exactly the ones where cross-version - # upgrade breakage is most expensive to discover at tag time. - pull_request: - branches: [ main, 'release/**', 'hotfix/**' ] - # "Upgrade-relevant code" allowlist (spec-f7.md §2.4): kept identical - # (modulo each workflow's own file/script entries) across conformance.yml, - # migration-test.yml and cross-version-smoke.yml, and pinned by - # TestAdvisoryWorkflowPathFilters. - paths: - - '**.go' - - '!**_test.go' - - 'go.mod' - - 'go.sum' - - 'Makefile' - - '.buildflags' - # Embedded into the binary (go:embed in internal/storage/schema), so a - # migration-only change is an upgrade-relevant code change. - - 'internal/storage/schema/migrations/**' - - '.github/workflows/cross-version-smoke.yml' - - 'scripts/upgrade-smoke-test.sh' - - # Allow manual trigger with custom version count - workflow_dispatch: - inputs: - version_count: - description: 'Number of previous releases to test against' - required: false - default: '5' - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - BD_DISABLE_METRICS: "1" - BD_DISABLE_EVENT_FLUSH: "1" - -jobs: - # F7c: smoke's old 1-job-per-version matrix (up to 30 on tag push) folded - # into chunks of 5 versions each, resolved by the `versions` job below. Each - # chunk builds the candidate binary once and loops over - # upgrade-smoke-test.sh's own SMOKE_VERSIONS env var (which already - # self-reinvokes per version and collects failures, so a bad version in a - # chunk does not hide the rest). - smoke: - name: Upgrade smoke (chunk ${{ strategy.job-index }}) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 20 - - strategy: - fail-fast: false - matrix: - chunk: ${{ fromJson(needs.versions.outputs.chunks) }} - - needs: versions - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: ${{ runner.environment == 'github-hosted' }} - - # F7c review fix S5: this job's own key copy must match main.yml's - # seeder key byte-for-byte (TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers), - # which now keys by UTC calendar day instead of github.sha so main - # pushes don't churn a multi-GB cache entry on every commit; this step - # computes that same day value locally so the key expression below - # resolves (steps.* references only resolve within the same job). - - name: Compute cache date - id: cache-date - if: runner.environment == 'self-hosted' - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Blacksmith setup-go cache - if: runner.environment == 'self-hosted' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: | - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}- - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Compute chunk cache key - id: chunk - env: - CHUNK_JSON: ${{ toJson(matrix.chunk) }} - run: | - versions="$(jq -r '. | join(" ")' <<<"$CHUNK_JSON")" - hash="$(printf '%s' "$versions" | sha256sum | cut -c1-16)" - echo "versions=$versions" >> "$GITHUB_OUTPUT" - echo "hash=$hash" >> "$GITHUB_OUTPUT" - - # actions/cache (bare) auto-saves at job end on any key miss, which - # would let a same-repo PR that changes the chunk's version set write a - # binaries cache entry another PR (or a later run) could then read back - # without re-downloading or re-verifying it. Restore-only here; the - # matching save below is gated off pull_request so only push/dispatch - # runs can populate this cache (B2, F7c implementation report). - - name: Restore previous release binaries cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: smoke-binaries-${{ steps.chunk.outputs.hash }}-${{ runner.os }} - - - name: Build candidate binary - run: go build -tags gms_pure_go -o /tmp/bd-candidate ./cmd/bd - - - name: Run upgrade smoke tests - env: - CANDIDATE_BIN: /tmp/bd-candidate - SMOKE_VERSIONS: ${{ steps.chunk.outputs.versions }} - run: ./scripts/upgrade-smoke-test.sh - - - name: Save previous release binaries cache - # Allow-list the two triggers this workflow actually intends to - # write from (push, workflow_dispatch), rather than deny-listing - # pull_request: a deny-list of `!= 'pull_request'` also admits a - # hypothetical future merge_group event, which this job's own - # runs-on expression already resolves to Blacksmith for - exactly - # the same same-repo-writer risk X1 closed for pull_request. Pinned - # by TestBlacksmithReachableAdvisoryJobsNeverSaveACache, which checks - # both the pull_request and merge_group trust contexts independently. - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: smoke-binaries-${{ steps.chunk.outputs.hash }}-${{ runner.os }} - - versions: - name: Resolve versions to test - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 5 - outputs: - chunks: ${{ steps.resolve.outputs.chunks }} - steps: - - name: Resolve release versions - id: resolve - env: - GH_TOKEN: ${{ github.token }} - # Tag push and manual dispatch test more versions; PRs test fewer for speed - VERSION_COUNT: ${{ (github.event_name == 'push' && '30') || (github.event_name == 'workflow_dispatch' && inputs.version_count) || '5' }} - run: | - VERSIONS=$(gh release list \ - --repo gastownhall/beads \ - --limit "$VERSION_COUNT" \ - --json tagName \ - --jq '[.[].tagName]') - # Chunks of 5: PRs (5 versions) get 1 chunk; tag pushes (30 - # versions) get 6. Folds the old 1-job-per-version matrix down to - # one job per chunk in `smoke` above, without dropping any version. - CHUNKS=$(jq -c '[range(0; length; 5) as $i | .[$i:$i+5]]' <<<"$VERSIONS") - echo "chunks=$CHUNKS" >> "$GITHUB_OUTPUT" - echo "Testing versions: $VERSIONS" - echo "Chunks: $CHUNKS" diff --git a/BUILD.bazel b/BUILD.bazel index 9916ecbfd..1cb1bcf75 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -24,6 +24,7 @@ exports_files( visibility = [ "//scripts/pr-lint:__pkg__", "//scripts/prlintmake:__pkg__", + "//tests/upgrade_smoke:__pkg__", ], ) @@ -376,6 +377,7 @@ filegroup( "//test/testmainconvention:repo_files", "//tests/migration:repo_files", "//tests/regression:repo_files", + "//tests/upgrade_smoke:repo_files", "//third_party/cc_toolchain:repo_files", "//third_party/patches:repo_files", "//tools/bazel:repo_files", diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index a6bac3a6d..259324933 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -497,6 +497,7 @@ go_binary( "//test/conformance:__pkg__", "//tests/migration:__pkg__", "//tests/regression:__pkg__", + "//tests/upgrade_smoke:__pkg__", ], ) diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index e1d25ed7b..78218e20f 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -157,9 +157,11 @@ Current PR-related workflow names: PR, merge group and push to `main` (results cached until its inputs change), against the catalog-pinned `@bd_releases//:v0.49.6` baseline and a hermetic dolt sql-server. -- `.github/workflows/cross-version-smoke.yml`: `Cross-Version Smoke Tests` - Runs on every PR to `main`, tag pushes, and manual dispatch. Does not - currently run on `merge_group`. +- The upgrade smoke tests (`scripts/upgrade-smoke-test.sh`, formerly + `cross-version-smoke.yml`) are `//tests/upgrade_smoke:upgrade_smoke__test`, + one per release in the newest 30 the release catalog pins, which + `bazel.yml`'s `bazel-test` lane runs on every PR, merge group and push to + `main` (cached until their inputs change). - `.github/workflows/nix-build.yml`: `nix build` Uses workflow-level `paths` filters on `pull_request` and `push`. This workflow must not be directly required. @@ -339,7 +341,8 @@ PR. It runs in the default branch's cache scope, which `push` runs on binaries); - the `beads-go-build-v2-*` GOCACHE entries restored by `main.yml` and `pr.yml`; - - the executables in `smoke-binaries-*` and `historical-dolt-*`. + - the executables in `smoke-binaries-*` and `historical-dolt-*` (both + retired with their workflows). GitHub's read-only default covers all of them. Hardening `release.yml` with `cache: false` is tracked separately. @@ -660,10 +663,6 @@ Do not require these existing check names directly: - `Test (Windows - smoke)` - `PR Lint (native)`, `PR Lint (windows)` and `PR Lint (darwin)` - `Test Nix Flake` -- `Upgrade smoke (chunk N)` (F7c: folded from one job per version into one job - per 5-version chunk; still never require a matrix-expanded chunk job - directly) -- `Resolve versions to test` - `Bazel / test` and the other jobs of `bazel.yml` - `Bazel Farm / *` (`bazel-farm.yml`'s advisory, PR-controlled results) @@ -1386,13 +1385,14 @@ PRs that don't touch Nix or Go module files. ### Cross-Version Smoke -`Cross-Version Smoke Tests` should remain non-required for ordinary PRs unless -maintainers explicitly choose to pay that cost in the aggregate gate. If it -becomes required, add `merge_group` and put it behind a detector plus aggregate -inside the required topology. Do not require matrix-expanded -`Upgrade smoke (chunk N)` jobs directly (F7c folded the old one-job-per-version -matrix into one job per 5-version chunk; the per-chunk job name changed but -the "do not require individually" guidance is unchanged). +`cross-version-smoke.yml` is retired. Its suite runs as +`//tests/upgrade_smoke/...` inside `bazel-test`, which `CI Gate / Required` +requires, so it now gates PRs and merge groups through the aggregate. Each +target pins its previous release from the release catalog (`@bd_releases`) +instead of resolving `gh release list` at run time, so the tested set is +reviewed: the newest 30 stable releases with a linux/amd64 asset, where the +workflow tested the newest 5 GitHub releases (prereleases included) on PRs +and 30 on tag pushes. ## Merge Queue diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index 18a805381..202b0dd33 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -200,7 +200,7 @@ Key jobs preserved by display name: | Workflow | Triggers | Main validation | |---|---|---| | `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (8 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | -| `cross-version-smoke.yml` | Tags, PRs, manual | PRs test latest 5 releases, tags test latest 30, via `scripts/upgrade-smoke-test.sh`. | +| `bazel.yml` `bazel-test` (formerly `cross-version-smoke.yml`) | PR, merge group, push to `main` | `//tests/upgrade_smoke:upgrade_smoke__test` for the newest 30 catalog-pinned releases, via `scripts/upgrade-smoke-test.sh` with `PREV_BIN`; cached until their inputs change. | | `bazel.yml` `bazel-test` (formerly `migration-test.yml`) | PR, merge group, push to `main` | `//tests/migration:historical_upgrade__test`, one target per reviewed release (14, v0.9.1 source-built offline) plus `legacy_bridge_test`, against catalog-pinned release binaries; cached until their inputs change. Local entrypoint stays `scripts/migration-test/run.sh`. | | `nightly.yml` | Daily schedule, manual | `go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./...` with `BEADS_TEST_SKIP=dolt`; checks coverage >= 30%. | | `nix-build.yml` | Push to `main` paths for Nix or Go module files, manual | `nix build .#default --print-build-logs`; no longer runs on `pull_request` (F7c, spec-f7.md §2.4) since PR Risk's required `test-nix` job (`nix run .#default` plus `nix flake check -L`) is a superset. | diff --git a/engdocs/ICU-POLICY.md b/engdocs/ICU-POLICY.md index 5939860d6..4eac25f12 100644 --- a/engdocs/ICU-POLICY.md +++ b/engdocs/ICU-POLICY.md @@ -57,7 +57,7 @@ Every build path that produces a binary for users must include `-tags gms_pure_g | macOS release | `.github/workflows/release.yml` | | Migration tests | `//tests/migration/...` (Bazel, `bazel.yml` `bazel-test`) | | Nightly tests | `.github/workflows/nightly.yml` | -| Cross-version smoke | `.github/workflows/cross-version-smoke.yml` | +| Cross-version smoke | `//tests/upgrade_smoke/...` (Bazel, `bazel.yml` `bazel-test`) | | Regression tests | `//tests/regression:regression_test` (Bazel, `bazel.yml` `bazel-cmd-dolt`) | ### Canonical pattern: source `.buildflags` diff --git a/internal/httpclient/wire/skew_test.go b/internal/httpclient/wire/skew_test.go index bf431d7c8..04b023171 100644 --- a/internal/httpclient/wire/skew_test.go +++ b/internal/httpclient/wire/skew_test.go @@ -31,7 +31,7 @@ import ( // // The split is the design's (test-lane map, the skew rows): the deterministic, // in-process half runs at PR cadence here, and the released-binary composition -// half runs in cross-version-smoke.yml. Neither substitutes for the other — +// half runs in //tests/upgrade_smoke. Neither substitutes for the other — // this half owns the branches, that half owns the reality check. // // Three skew directions are covered, and they fail in different ways: diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index b4efd8902..443631a45 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -125,6 +125,11 @@ filegroup( visibility = ["//tests/migration:__pkg__"], ) +exports_files( + ["upgrade-smoke-test.sh"], + visibility = ["//tests/upgrade_smoke:__pkg__"], +) + exports_files( [ "ci/fmt-check.sh", diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 100d2e776..517220025 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -1,10 +1,7 @@ package scripts_test import ( - "encoding/json" "fmt" - "os/exec" - "regexp" "sort" "strings" "testing" @@ -13,49 +10,13 @@ import ( ) // F7c (spec-f7.md §2.4, §4.3): advisory workflows moved onto a same-repo-PR -// Blacksmith runner, gained a shared "upgrade-relevant code" path filter, and -// had their matrices folded (Migration Test Harness 14 -> 3, Cross-Version -// Smoke 6 -> 2). These tests pin the invariants that make those changes safe: -// the path filter is identical where it should be, no historical version or -// scenario was dropped by a fold, no advisory job can read a secret just -// because it now names a Blacksmith label, and the Blacksmith-side setup-go -// seed in main.yml actually exists for the jobs that depend on it. - -// advisoryPathFilteredWorkflows are the workflows that share the -// "upgrade-relevant code" allowlist verbatim, save for each one's own -// workflow-file and script entries (spec-f7.md §2.4). conformance.yml and -// migration-test.yml were the others until their tiers moved to Bazel -// (embeddeddolt_conformance_*, //test/conformance:conformance_test, -// //tests/migration). -var advisoryPathFilteredWorkflows = []string{ - "cross-version-smoke.yml", -} - -// advisoryPathFilterBase is the shared prefix of the allowlist: any non-test -// Go change, build input, or embedded schema migration. It must appear, in -// this order, at the start of each of advisoryPathFilteredWorkflows' -// pull_request.paths list. -var advisoryPathFilterBase = []string{ - "**.go", - "!**_test.go", - "go.mod", - "go.sum", - "Makefile", - ".buildflags", - // The .up.sql files are go:embed'd by internal/storage/schema, so they - // change upgrade behavior without matching "**.go"; a migration that - // ships with only a _test.go beside it would otherwise skip all three. - "internal/storage/schema/migrations/**", -} - -// advisoryPathFilterOwnEntries is each workflow's own file/script additions, -// appended after advisoryPathFilterBase. -var advisoryPathFilterOwnEntries = map[string][]string{ - "cross-version-smoke.yml": { - ".github/workflows/cross-version-smoke.yml", - "scripts/upgrade-smoke-test.sh", - }, -} +// Blacksmith runner. These tests pin the invariants that make that safe: no +// advisory job can read a secret just because it now names a Blacksmith +// label, and the Blacksmith-side setup-go seed in main.yml actually exists +// for the jobs that depend on it. The path-filtered upgrade suites F7c folded +// (Migration Test Harness, Cross-Version Smoke) and their filter and fold +// checks are gone: those suites run under Bazel (//tests/migration, +// //tests/upgrade_smoke). type pullRequestPaths struct { On struct { @@ -93,42 +54,6 @@ func readPushPaths(t *testing.T, file string) []string { return parsed.On.Push.Paths } -// TestAdvisoryWorkflowPathFiltersAreIdentical pins that the shared base of -// the "upgrade-relevant code" allowlist is byte-for-byte identical, in the -// same order, across all three workflows it applies to. GitHub Actions has no -// cross-file include for `on:` triggers, so this is the fallback the spec -// explicitly allows: identical literal lists plus a policy test asserting -// they stay identical (spec-f7.md §2.4). -func TestAdvisoryWorkflowPathFiltersAreIdentical(t *testing.T) { - for _, file := range advisoryPathFilteredWorkflows { - paths := readPullRequestPaths(t, file) - if len(paths) < len(advisoryPathFilterBase) { - t.Fatalf("%s pull_request.paths = %v, too short to hold the shared base %v", file, paths, advisoryPathFilterBase) - } - got := paths[:len(advisoryPathFilterBase)] - for i, want := range advisoryPathFilterBase { - if got[i] != want { - t.Errorf("%s pull_request.paths[%d] = %q, want %q (shared base must match byte-for-byte and in order)", file, i, got[i], want) - } - } - } -} - -// TestAdvisoryWorkflowPathFiltersCoverOwnInputs pins that each workflow also -// allowlists its own workflow file and the scripts/fixtures it actually -// exercises, so an edit to e.g. scripts/upgrade-smoke-test.sh is never silently -// skipped by the filter that was added to cut unrelated-PR load. -func TestAdvisoryWorkflowPathFiltersCoverOwnInputs(t *testing.T) { - for file, want := range advisoryPathFilterOwnEntries { - paths := readPullRequestPaths(t, file) - for _, entry := range want { - if !contains(paths, entry) { - t.Errorf("%s pull_request.paths %v does not contain its own entry %q", file, paths, entry) - } - } - } -} - // TestNixBuildDropsPullRequestTriggerNotPushOrDispatch pins the one // "delete the pull_request trigger" trigger change in F7c: nix-build.yml's // PR coverage is fully redundant with PR Risk's required test-nix job (which @@ -199,137 +124,11 @@ func sortedCopy(items []string) []string { return out } -// --- Cross-Version Smoke: 6 -> 2 jobs, chunks of 5, no version dropped ---- - -// ghExtractJQProgram pulls the single-quoted jq program out of a standalone -// `jq - ''` invocation inside a step's `run:` text, so a test -// can execute the REAL program with `jq` directly instead of asserting on a -// substring of the surrounding bash. The flag group is required (one or -// more) specifically so this does not also match `gh`'s own `--jq -// '[.[].tagName]'` filter, which has no `-c`/`-r` flag of its own between -// "jq" and the quoted program. None of this repo's embedded jq programs -// contain a literal single quote, so a non-greedy single-quote match is -// sufficient. -var jqProgramPattern = regexp.MustCompile(`jq(?: -[A-Za-z]+)+ '([^']*)'`) - -func ghExtractJQProgram(t *testing.T, run string) string { - t.Helper() - m := jqProgramPattern.FindStringSubmatch(run) - if m == nil { - t.Fatalf("no `jq '...'` invocation found in:\n%s", run) - } - return m[1] -} - -func runJQ(t *testing.T, program string, stdin string) string { - t.Helper() - requireHostTool(t, "jq") - cmd := exec.Command("jq", "-c", program) - cmd.Stdin = strings.NewReader(stdin) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("jq -c %q <<<%q: %v\n%s", program, stdin, err, out) - } - return strings.TrimSpace(string(out)) -} - -// TestCrossVersionSmokeChunksEveryResolvedVersion runs the REAL jq programs -// embedded in the "Resolve release versions" and "Compute chunk cache key" -// steps against synthetic version lists, instead of asserting on their -// source text (F7c review fix S1, closes mutation M6 `[range(0; length; 5) -// as $i | .[$i:$i+4]]` off-by-one and M7 jq-slice-dropping-versions -// mutations): every resolved version must appear in exactly one chunk, no -// chunk may exceed 5 versions, and the chunk-key step's join must reproduce -// every version in its chunk, in order, space-separated. -func TestCrossVersionSmokeChunksEveryResolvedVersion(t *testing.T) { - workflow := readCIWorkflow(t, "cross-version-smoke.yml") - - versionsJob := workflow.job(t, "versions") - resolve := versionsJob.step(t, "Resolve release versions") - chunkProgram := ghExtractJQProgram(t, resolve.Run) - if versionsJob.Outputs["chunks"] == "" { - t.Errorf("cross-version-smoke.yml's versions job has no chunks output") - } - - for _, n := range []int{0, 1, 4, 5, 6, 9, 10, 29, 30} { - versions := make([]string, 0, n) - for i := 0; i < n; i++ { - versions = append(versions, fmt.Sprintf("v0.%d.0", i)) - } - versionsJSON, err := json.Marshal(versions) - if err != nil { - t.Fatal(err) - } - var chunks [][]string - if err := json.Unmarshal([]byte(runJQ(t, chunkProgram, string(versionsJSON))), &chunks); err != nil { - t.Fatalf("n=%d: chunk output did not parse as [][]string: %v", n, err) - } - wantChunks := (n + 4) / 5 - if n == 0 { - wantChunks = 0 - } - if len(chunks) != wantChunks { - t.Errorf("n=%d: got %d chunks, want %d", n, len(chunks), wantChunks) - } - var flat []string - for _, c := range chunks { - if len(c) > 5 { - t.Errorf("n=%d: chunk %v has more than 5 versions", n, c) - } - flat = append(flat, c...) - } - if !equalStrings(flat, versions) { - t.Errorf("n=%d: concatenated chunks = %v, want exactly the resolved version list %v in order", n, flat, versions) - } - } - - smokeJob := workflow.job(t, "smoke") - if !contains(smokeJob.Needs, "versions") { - t.Errorf("cross-version-smoke.yml's smoke job does not need versions") - } - - chunkKeyStep := smokeJob.step(t, "Compute chunk cache key") - joinProgram := ghExtractJQProgram(t, chunkKeyStep.Run) - for _, chunk := range [][]string{ - {"v1.2.2"}, - {"v1.0.0", "v1.0.1", "v1.1.0", "v1.1.2", "v1.2.2"}, - } { - chunkJSON, err := json.Marshal(chunk) - if err != nil { - t.Fatal(err) - } - got := strings.Trim(runJQ(t, joinProgram, string(chunkJSON)), `"`) - want := strings.Join(chunk, " ") - if got != want { - t.Errorf("chunk-key join(%v) = %q, want %q", chunk, got, want) - } - } - - buildStep := smokeJob.stepIndex(t, "Build candidate binary") - runStep := smokeJob.stepIndex(t, "Run upgrade smoke tests") - if buildStep >= runStep { - t.Errorf("Build candidate binary (index %d) must run before Run upgrade smoke tests (index %d), so the candidate is built once per chunk, not once per version", buildStep, runStep) - } - - run := smokeJob.step(t, "Run upgrade smoke tests") - if run.Env["SMOKE_VERSIONS"] != "${{ steps.chunk.outputs.versions }}" { - t.Errorf("cross-version-smoke.yml's smoke job SMOKE_VERSIONS = %q, want the chunk step's versions output", run.Env["SMOKE_VERSIONS"]) - } - // Exact match (not a substring check, F7c review fix S1): the whole point - // of SMOKE_VERSIONS is that upgrade-smoke-test.sh's own loop consumes it; - // any extra positional arg (e.g. a reintroduced matrix.prev_version) would - // silently make the script test only one version per chunk again. - if want := "./scripts/upgrade-smoke-test.sh"; strings.TrimSpace(run.Run) != want { - t.Errorf("cross-version-smoke.yml's Run upgrade smoke tests run = %q, want exactly %q", run.Run, want) - } -} - // --- Security: no job gains a secret just by naming a Blacksmith label ---- // blacksmithAdvisoryWorkflows are every workflow file F7c moved a job onto a // same-repo-PR (or push-only, for main.yml's seed) Blacksmith runner. var blacksmithAdvisoryWorkflows = []string{ - "cross-version-smoke.yml", "docs-mintlify.yml", "main.yml", } @@ -379,8 +178,6 @@ var advisoryBlacksmithRunnerJobs = []struct { blacksmithLabel string fallback string }{ - {"cross-version-smoke.yml", "smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, - {"cross-version-smoke.yml", "versions", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, {"docs-mintlify.yml", "broken-links", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, } @@ -446,16 +243,13 @@ func TestF7cAdvisorySameRepoBlacksmithExpressionSemantics(t *testing.T) { // blacksmithSetupGoCacheConsumers is every advisory job that restores the // self-defined `blacksmith-sg-v1-` setup-go cache main.yml's // blacksmith-setup-go-cache job seeds (B2, F7c implementation report). -var blacksmithSetupGoCacheConsumers = map[string][]string{ - "cross-version-smoke.yml": {"smoke"}, -} +var blacksmithSetupGoCacheConsumers = map[string][]string{} // blacksmithSetupGoCacheKeyNamespace is the self-defined cache key prefix // (not setup-go's own implicit key) that the seeder and every consumer share, // so the "no save in a consumer job" checks below can scope to exactly this -// cache without also flagging an unrelated, legitimately-caching step (a -// binary cache with a distinct restore/save pair of its own - see -// advisoryBinaryCaches below). +// cache without also flagging an unrelated, legitimately-caching step in +// another namespace. const blacksmithSetupGoCacheKeyNamespace = "blacksmith-sg-v1-" // TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers pins that main.yml's @@ -636,10 +430,10 @@ func TestMainWorkflowHasNoSameRepoPRReachableTrigger(t *testing.T) { // main.yml's seeder may ever write it, so a same-repo PR run can read the // cache but never poison what another PR or main's seeder reads back. The // "no save" check is scoped to the blacksmith-sg-v1- key namespace -// specifically (not "no actions/cache/save in this job at all"), since a -// binary cache in a different namespace is governed by its own -// restore/save-gated pair below (advisoryBinaryCaches / F7c review fix X1), -// not an exemption from this one. +// specifically (not "no actions/cache/save in this job at all"): a cache in +// another namespace is the general sweep's concern below +// (TestBlacksmithReachableAdvisoryJobsNeverSaveACache), not an exemption +// from this one. func TestAdvisoryBlacksmithConsumersAreCacheRestoreOnly(t *testing.T) { for file, jobNames := range blacksmithSetupGoCacheConsumers { workflow := readCIWorkflow(t, file) @@ -757,81 +551,6 @@ func TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers(t *testing.T) { } } -// --- Binary caches: restore-always, save only off pull_request ------------ - -// advisoryBinaryCaches are the per-binary caches F7c review fixes B2 and X1 -// converted from a monolithic (auto-saving) actions/cache into an explicit -// restore/save pair, so a same-repo PR can read a previously published -// binary but never publish its own into a cache another run would trust -// unverified. migration-test.yml's historical-dolt-* cache was originally -// exempted here on the theory that scripts/migration-test/lib/binary.sh's -// sha256 verification of every extracted archive made a bare, auto-saving -// actions/cache safe regardless of who wrote it. That exemption was unsound -// (F7c review fix X1): `tar -P` extraction plus lib/binary.sh's `cp -f` -// following a symlink planted inside the archive can redirect the final copy -// to an arbitrary path (e.g. over the checked-out workspace or the candidate -// binary itself) before the checksum check ever runs, so checksum -// verification alone does not make a same-repo-PR-writable cache entry safe -// to trust. historical-dolt-* now gets the same restore-always/save-off-PR -// split as the other two. -var advisoryBinaryCaches = []struct { - file, job, restoreStep, saveStep, keyPrefix string - // wantSaveIf is the save step's `if:` condition, required byte-for-byte - // (F7c review fix S3): a `strings.Contains` check here would pass under - // e.g. `... || true`, which always evaluates true and silently - // reintroduces the same-repo-PR poisoning path this whole table exists - // to close. - wantSaveIf string -}{ - {"cross-version-smoke.yml", "smoke", "Restore previous release binaries cache", "Save previous release binaries cache", "smoke-binaries-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"}, -} - -// TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated pins the restore/save -// split itself (F7c review fix B2): the restore step always runs (modulo the -// job's own pre-existing gate, if any), -// the save step additionally requires an exact allow-list of -// `github.event_name == 'push' || github.event_name == 'workflow_dispatch'` -// (F7c review fix, discovered via the S2 sweep: a deny-list of -// `!= 'pull_request'` also admits a hypothetical future merge_group event, -// which each of these three jobs' runs-on expressions already resolve to -// Blacksmith for), both steps key off the same cache, and no monolithic -// (bare) actions/cache step remains for either binary cache - a monolithic -// step would silently reintroduce the auto-save-on-any-PR poisoning path B2 -// closes. -func TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated(t *testing.T) { - for _, c := range advisoryBinaryCaches { - t.Run(c.file, func(t *testing.T) { - workflow := readCIWorkflow(t, c.file) - job := workflow.job(t, c.job) - - restore := job.step(t, c.restoreStep) - if actionFamily(restore.Uses) != cacheRestoreActionFamily { - t.Errorf("%s job %s step %q uses %q, want family %q", c.file, c.job, c.restoreStep, restore.Uses, cacheRestoreActionFamily) - } - if !strings.HasPrefix(restore.With["key"], c.keyPrefix) { - t.Errorf("%s job %s step %q key = %q, want prefix %q", c.file, c.job, c.restoreStep, restore.With["key"], c.keyPrefix) - } - - save := job.step(t, c.saveStep) - if actionFamily(save.Uses) != cacheSaveActionFamily { - t.Errorf("%s job %s step %q uses %q, want family %q", c.file, c.job, c.saveStep, save.Uses, cacheSaveActionFamily) - } - if save.If != c.wantSaveIf { - t.Errorf("%s job %s step %q has if=%q, want exactly %q", c.file, c.job, c.saveStep, save.If, c.wantSaveIf) - } - if save.With["key"] != restore.With["key"] { - t.Errorf("%s job %s: restore key %q != save key %q", c.file, c.job, restore.With["key"], save.With["key"]) - } - - for _, step := range job.Steps { - if actionFamily(step.Uses) == cacheMonolithicActionFamily && strings.HasPrefix(step.With["key"], c.keyPrefix) { - t.Errorf("%s job %s has a monolithic actions/cache step keyed %q; B2 requires an explicit restore/save split here", c.file, c.job, step.With["key"]) - } - } - }) - } -} - // --- General sweep: no Blacksmith-reachable advisory job may save a cache -- // generalCacheSweepWorkflows is every F7c advisory workflow plus main.yml diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index b3f0980c6..ae88cefb7 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -126,7 +126,7 @@ func TestPullRequestWorkflowsTriggerOnHotfixBranches(t *testing.T) { } root := sourceRepoRoot(t) for _, name := range []string{ - "pr.yml", prRiskWorkflowName, "cross-version-smoke.yml", + "pr.yml", prRiskWorkflowName, } { t.Run(name, func(t *testing.T) { var doc triggers @@ -2709,9 +2709,8 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && // F7c: used to define its own sameRepoBlacksmith4vcpu here (same expression // as sameRepoBlacksmith2vcpu with the 4 vCPU label, for advisory jobs that -// compile Go: cross-version-smoke.yml). F7a independently defined the same -// const; both are now served by the single shared sameRepoBlacksmith4vcpu in -// ci_blacksmith_runner_test.go. +// compile Go). F7a independently defined the same const; both are now served +// by the single shared sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go. // F7b: venue-matrix runs-on for main.yml jobs that seed a Blacksmith GOCACHE // alongside their GitHub-hosted seed ("venue: [blacksmith, github]"): @@ -3038,8 +3037,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // is cheap enough to size at 2 vCPU per spec-f7.md §2.2) and moves // to Blacksmith for same-repo PRs/merge_group only; forks, // Dependabot and push stay on ubuntu-latest. - "cross-version-smoke.yml": {"smoke": sameRepoBlacksmith4vcpu, "versions": sameRepoBlacksmith2vcpu}, - "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, + "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, // main.yml's seeder job (B2, F7c implementation report) is the one // Blacksmith job that is NOT gated by the same-repo-PR expression: it // is push-to-main only (always trusted), so it wraps the literal @@ -3103,7 +3101,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // Blacksmith label beyond the jobs listed in `want` above. for _, file := range []string{ "pr.yml", "pr-risk.yml", "main.yml", - "cross-version-smoke.yml", "docs-mintlify.yml", + "docs-mintlify.yml", } { workflow := readCIWorkflow(t, file) allowed := want[file] diff --git a/scripts/upgrade-smoke-test.sh b/scripts/upgrade-smoke-test.sh index d5d426b4f..52b05fbf2 100755 --- a/scripts/upgrade-smoke-test.sh +++ b/scripts/upgrade-smoke-test.sh @@ -103,6 +103,17 @@ case "$ARCH" in esac get_previous_binary() { + # PREV_BIN (set by //tests/upgrade_smoke's targets) is the previous + # release already pinned and sha256-verified against the release catalog, + # so nothing is downloaded; it must report PREV_VERSION. + if [ -n "${PREV_BIN:-}" ]; then + if ! "$PREV_BIN" version 2>/dev/null | grep -qF " ${PREV_VER_BARE} "; then + echo -e "${RED}PREV_BIN=${PREV_BIN} does not report ${PREV_VERSION}${NC}" >&2 + exit 1 + fi + echo "$PREV_BIN" + return + fi local cached="$CACHE_DIR/bd-${PREV_VER_BARE}" if [ -x "$cached" ]; then echo "$cached" diff --git a/tests/upgrade_smoke/BUILD.bazel b/tests/upgrade_smoke/BUILD.bazel new file mode 100644 index 000000000..be70457de --- /dev/null +++ b/tests/upgrade_smoke/BUILD.bazel @@ -0,0 +1,43 @@ +load(":defs.bzl", "upgrade_smoke_tests") + +# The upgrade smoke tests (formerly cross-version-smoke.yml): one target per +# previous release, upgrade_smoke__test, running +# scripts/upgrade-smoke-test.sh from the pinned release +# (@bd_releases, sha256-verified against the release catalog) to the +# non-race bd_for_tests. The releases are the newest `window` the catalog +# pins a linux/amd64 asset for (its stable GitHub releases), so a release +# added to the catalog joins the window and the oldest drops out. The PR-core +# lane (bazel-test) runs them remotely on every PR, merge group and push, and +# a target re-runs only when the candidate, the script or its release +# changes. +# +# Tags: +# pr-core-only: shell tests against the release-shaped bd_for_tests (no +# integration tag); the integration lane, which requires Go testcases, +# leaves them out. +upgrade_smoke_tests( + tags = ["pr-core-only"], + window = 30, +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tests/upgrade_smoke/defs.bzl b/tests/upgrade_smoke/defs.bzl new file mode 100644 index 000000000..3c3045e56 --- /dev/null +++ b/tests/upgrade_smoke/defs.bzl @@ -0,0 +1,32 @@ +"""One upgrade smoke test per pinned previous release (see BUILD.bazel).""" + +load("@bd_releases//:versions.bzl", "VERSIONS") +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") + +def upgrade_smoke_tests(window, tags): + """Declares upgrade_smoke__test for the newest `window` catalog releases. + + Args: + window: how many of the newest pinned releases to test. + tags: the tests' tags. + """ + for v in VERSIONS[-window:]: + sh_test( + name = "upgrade_smoke_{}_test".format(v.replace(".", "_")), + size = "large", + srcs = ["upgrade_smoke_test.sh"], + args = [v], + data = [ + "//:.buildflags", + "//cmd/bd:bd_for_tests", + "//scripts:upgrade-smoke-test.sh", + "@bd_releases//:" + v, + ], + env = { + "CANDIDATE_BIN": "$(rootpath //cmd/bd:bd_for_tests)", + "PREV_BIN": "$(rootpath @bd_releases//:{})".format(v), + }, + tags = tags, + target_compatible_with = LINUX_AMD64, + ) diff --git a/tests/upgrade_smoke/upgrade_smoke_test.sh b/tests/upgrade_smoke/upgrade_smoke_test.sh new file mode 100755 index 000000000..24130e901 --- /dev/null +++ b/tests/upgrade_smoke/upgrade_smoke_test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# One previous release of the upgrade smoke tests (formerly +# cross-version-smoke.yml's `scripts/upgrade-smoke-test.sh` loop): that +# release → the candidate, with both binaries pinned inputs, so the script +# downloads and builds nothing. +# +# The first argument is the release. CANDIDATE_BIN and PREV_BIN are +# runfiles-relative paths ($(rootpath ...), relative to the working directory +# Bazel starts the test in), made absolute here before the script changes +# directory. Any further arguments (a lane's --test_arg flags, meant for Go +# test binaries) are ignored. +set -euo pipefail +release="$1" +CANDIDATE_BIN="$PWD/${CANDIDATE_BIN:?}" +PREV_BIN="$PWD/${PREV_BIN:?}" +export CANDIDATE_BIN PREV_BIN +exec scripts/upgrade-smoke-test.sh "$release"