From ec2644a2f2bd8238cc2cc4ffa781534124cda6dd Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 00:27:56 +0000 Subject: [PATCH] ci(bazel): run the upgrade smoke tests under Bazel; retire cross-version-smoke.yml cross-version-smoke.yml resolved the newest 5 GitHub releases (30 on a tag push) with `gh release list`, then built the candidate and downloaded each release on a GitHub runner for scripts/upgrade-smoke-test.sh. //tests/upgrade_smoke has one target per release in the newest 30 the release catalog pins a linux/amd64 asset for (@bd_releases//:versions.bzl): upgrade_smoke__test runs the script against the non-race bd_for_tests with that release pinned as PREV_BIN (sha256-verified when Bazel fetches it; the script checks it reports the release and downloads nothing). A release added to the catalog joins the window. Tagged pr-core-only, so bazel-test (not the integration lane) runs them remotely on every PR, merge group and push, cached until the candidate, the script or the release changes. The tested set is now reviewed rather than resolved at run time: stable releases from the catalog, where the workflow took GitHub releases including prereleases. cross-version-smoke.yml and the F7c checks of its chunk fold, path filter, runners and binary cache are removed (the path-filter and binary-cache checks had no other workflow left). Refs: ga-96smfk.19 Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/cross-version-smoke.yml | 175 ------------ BUILD.bazel | 2 + cmd/bd/BUILD.bazel | 1 + engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 30 +-- engdocs/CI_TEST_SURFACE_AUDIT.md | 2 +- engdocs/ICU-POLICY.md | 2 +- internal/httpclient/wire/skew_test.go | 2 +- scripts/BUILD.bazel | 5 + scripts/ci_f7c_advisory_test.go | 309 +--------------------- scripts/ci_workflow_test.go | 12 +- scripts/upgrade-smoke-test.sh | 11 + tests/upgrade_smoke/BUILD.bazel | 43 +++ tests/upgrade_smoke/defs.bzl | 32 +++ tests/upgrade_smoke/upgrade_smoke_test.sh | 17 ++ 14 files changed, 148 insertions(+), 495 deletions(-) delete mode 100644 .github/workflows/cross-version-smoke.yml create mode 100644 tests/upgrade_smoke/BUILD.bazel create mode 100644 tests/upgrade_smoke/defs.bzl create mode 100755 tests/upgrade_smoke/upgrade_smoke_test.sh diff --git a/.github/workflows/cross-version-smoke.yml b/.github/workflows/cross-version-smoke.yml deleted file mode 100644 index 938410be8..000000000 --- a/.github/workflows/cross-version-smoke.yml +++ /dev/null @@ -1,175 +0,0 @@ -name: Cross-Version Smoke Tests - -on: - # On tag push (release gate): test the last 30 releases for broad regression coverage - push: - tags: - - 'v*' - - # On PR to main or a release branch: test only the last 5 releases for fast - # feedback. Release-targeted PRs are exactly the ones where cross-version - # upgrade breakage is most expensive to discover at tag time. - pull_request: - branches: [ main, 'release/**', 'hotfix/**' ] - # "Upgrade-relevant code" allowlist (spec-f7.md §2.4): kept identical - # (modulo each workflow's own file/script entries) across conformance.yml, - # migration-test.yml and cross-version-smoke.yml, and pinned by - # TestAdvisoryWorkflowPathFilters. - paths: - - '**.go' - - '!**_test.go' - - 'go.mod' - - 'go.sum' - - 'Makefile' - - '.buildflags' - # Embedded into the binary (go:embed in internal/storage/schema), so a - # migration-only change is an upgrade-relevant code change. - - 'internal/storage/schema/migrations/**' - - '.github/workflows/cross-version-smoke.yml' - - 'scripts/upgrade-smoke-test.sh' - - # Allow manual trigger with custom version count - workflow_dispatch: - inputs: - version_count: - description: 'Number of previous releases to test against' - required: false - default: '5' - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - BD_DISABLE_METRICS: "1" - BD_DISABLE_EVENT_FLUSH: "1" - -jobs: - # F7c: smoke's old 1-job-per-version matrix (up to 30 on tag push) folded - # into chunks of 5 versions each, resolved by the `versions` job below. Each - # chunk builds the candidate binary once and loops over - # upgrade-smoke-test.sh's own SMOKE_VERSIONS env var (which already - # self-reinvokes per version and collects failures, so a bad version in a - # chunk does not hide the rest). - smoke: - name: Upgrade smoke (chunk ${{ strategy.job-index }}) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 20 - - strategy: - fail-fast: false - matrix: - chunk: ${{ fromJson(needs.versions.outputs.chunks) }} - - needs: versions - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: ${{ runner.environment == 'github-hosted' }} - - # F7c review fix S5: this job's own key copy must match main.yml's - # seeder key byte-for-byte (TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers), - # which now keys by UTC calendar day instead of github.sha so main - # pushes don't churn a multi-GB cache entry on every commit; this step - # computes that same day value locally so the key expression below - # resolves (steps.* references only resolve within the same job). - - name: Compute cache date - id: cache-date - if: runner.environment == 'self-hosted' - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Blacksmith setup-go cache - if: runner.environment == 'self-hosted' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: | - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}- - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Compute chunk cache key - id: chunk - env: - CHUNK_JSON: ${{ toJson(matrix.chunk) }} - run: | - versions="$(jq -r '. | join(" ")' <<<"$CHUNK_JSON")" - hash="$(printf '%s' "$versions" | sha256sum | cut -c1-16)" - echo "versions=$versions" >> "$GITHUB_OUTPUT" - echo "hash=$hash" >> "$GITHUB_OUTPUT" - - # actions/cache (bare) auto-saves at job end on any key miss, which - # would let a same-repo PR that changes the chunk's version set write a - # binaries cache entry another PR (or a later run) could then read back - # without re-downloading or re-verifying it. Restore-only here; the - # matching save below is gated off pull_request so only push/dispatch - # runs can populate this cache (B2, F7c implementation report). - - name: Restore previous release binaries cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: smoke-binaries-${{ steps.chunk.outputs.hash }}-${{ runner.os }} - - - name: Build candidate binary - run: go build -tags gms_pure_go -o /tmp/bd-candidate ./cmd/bd - - - name: Run upgrade smoke tests - env: - CANDIDATE_BIN: /tmp/bd-candidate - SMOKE_VERSIONS: ${{ steps.chunk.outputs.versions }} - run: ./scripts/upgrade-smoke-test.sh - - - name: Save previous release binaries cache - # Allow-list the two triggers this workflow actually intends to - # write from (push, workflow_dispatch), rather than deny-listing - # pull_request: a deny-list of `!= 'pull_request'` also admits a - # hypothetical future merge_group event, which this job's own - # runs-on expression already resolves to Blacksmith for - exactly - # the same same-repo-writer risk X1 closed for pull_request. Pinned - # by TestBlacksmithReachableAdvisoryJobsNeverSaveACache, which checks - # both the pull_request and merge_group trust contexts independently. - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: smoke-binaries-${{ steps.chunk.outputs.hash }}-${{ runner.os }} - - versions: - name: Resolve versions to test - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 5 - outputs: - chunks: ${{ steps.resolve.outputs.chunks }} - steps: - - name: Resolve release versions - id: resolve - env: - GH_TOKEN: ${{ github.token }} - # Tag push and manual dispatch test more versions; PRs test fewer for speed - VERSION_COUNT: ${{ (github.event_name == 'push' && '30') || (github.event_name == 'workflow_dispatch' && inputs.version_count) || '5' }} - run: | - VERSIONS=$(gh release list \ - --repo gastownhall/beads \ - --limit "$VERSION_COUNT" \ - --json tagName \ - --jq '[.[].tagName]') - # Chunks of 5: PRs (5 versions) get 1 chunk; tag pushes (30 - # versions) get 6. Folds the old 1-job-per-version matrix down to - # one job per chunk in `smoke` above, without dropping any version. - CHUNKS=$(jq -c '[range(0; length; 5) as $i | .[$i:$i+5]]' <<<"$VERSIONS") - echo "chunks=$CHUNKS" >> "$GITHUB_OUTPUT" - echo "Testing versions: $VERSIONS" - echo "Chunks: $CHUNKS" diff --git a/BUILD.bazel b/BUILD.bazel index 9916ecbfd..1cb1bcf75 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -24,6 +24,7 @@ exports_files( visibility = [ "//scripts/pr-lint:__pkg__", "//scripts/prlintmake:__pkg__", + "//tests/upgrade_smoke:__pkg__", ], ) @@ -376,6 +377,7 @@ filegroup( "//test/testmainconvention:repo_files", "//tests/migration:repo_files", "//tests/regression:repo_files", + "//tests/upgrade_smoke:repo_files", "//third_party/cc_toolchain:repo_files", "//third_party/patches:repo_files", "//tools/bazel:repo_files", diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index a6bac3a6d..259324933 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -497,6 +497,7 @@ go_binary( "//test/conformance:__pkg__", "//tests/migration:__pkg__", "//tests/regression:__pkg__", + "//tests/upgrade_smoke:__pkg__", ], ) diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index e1d25ed7b..78218e20f 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -157,9 +157,11 @@ Current PR-related workflow names: PR, merge group and push to `main` (results cached until its inputs change), against the catalog-pinned `@bd_releases//:v0.49.6` baseline and a hermetic dolt sql-server. -- `.github/workflows/cross-version-smoke.yml`: `Cross-Version Smoke Tests` - Runs on every PR to `main`, tag pushes, and manual dispatch. Does not - currently run on `merge_group`. +- The upgrade smoke tests (`scripts/upgrade-smoke-test.sh`, formerly + `cross-version-smoke.yml`) are `//tests/upgrade_smoke:upgrade_smoke__test`, + one per release in the newest 30 the release catalog pins, which + `bazel.yml`'s `bazel-test` lane runs on every PR, merge group and push to + `main` (cached until their inputs change). - `.github/workflows/nix-build.yml`: `nix build` Uses workflow-level `paths` filters on `pull_request` and `push`. This workflow must not be directly required. @@ -339,7 +341,8 @@ PR. It runs in the default branch's cache scope, which `push` runs on binaries); - the `beads-go-build-v2-*` GOCACHE entries restored by `main.yml` and `pr.yml`; - - the executables in `smoke-binaries-*` and `historical-dolt-*`. + - the executables in `smoke-binaries-*` and `historical-dolt-*` (both + retired with their workflows). GitHub's read-only default covers all of them. Hardening `release.yml` with `cache: false` is tracked separately. @@ -660,10 +663,6 @@ Do not require these existing check names directly: - `Test (Windows - smoke)` - `PR Lint (native)`, `PR Lint (windows)` and `PR Lint (darwin)` - `Test Nix Flake` -- `Upgrade smoke (chunk N)` (F7c: folded from one job per version into one job - per 5-version chunk; still never require a matrix-expanded chunk job - directly) -- `Resolve versions to test` - `Bazel / test` and the other jobs of `bazel.yml` - `Bazel Farm / *` (`bazel-farm.yml`'s advisory, PR-controlled results) @@ -1386,13 +1385,14 @@ PRs that don't touch Nix or Go module files. ### Cross-Version Smoke -`Cross-Version Smoke Tests` should remain non-required for ordinary PRs unless -maintainers explicitly choose to pay that cost in the aggregate gate. If it -becomes required, add `merge_group` and put it behind a detector plus aggregate -inside the required topology. Do not require matrix-expanded -`Upgrade smoke (chunk N)` jobs directly (F7c folded the old one-job-per-version -matrix into one job per 5-version chunk; the per-chunk job name changed but -the "do not require individually" guidance is unchanged). +`cross-version-smoke.yml` is retired. Its suite runs as +`//tests/upgrade_smoke/...` inside `bazel-test`, which `CI Gate / Required` +requires, so it now gates PRs and merge groups through the aggregate. Each +target pins its previous release from the release catalog (`@bd_releases`) +instead of resolving `gh release list` at run time, so the tested set is +reviewed: the newest 30 stable releases with a linux/amd64 asset, where the +workflow tested the newest 5 GitHub releases (prereleases included) on PRs +and 30 on tag pushes. ## Merge Queue diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index 18a805381..202b0dd33 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -200,7 +200,7 @@ Key jobs preserved by display name: | Workflow | Triggers | Main validation | |---|---|---| | `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (8 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | -| `cross-version-smoke.yml` | Tags, PRs, manual | PRs test latest 5 releases, tags test latest 30, via `scripts/upgrade-smoke-test.sh`. | +| `bazel.yml` `bazel-test` (formerly `cross-version-smoke.yml`) | PR, merge group, push to `main` | `//tests/upgrade_smoke:upgrade_smoke__test` for the newest 30 catalog-pinned releases, via `scripts/upgrade-smoke-test.sh` with `PREV_BIN`; cached until their inputs change. | | `bazel.yml` `bazel-test` (formerly `migration-test.yml`) | PR, merge group, push to `main` | `//tests/migration:historical_upgrade__test`, one target per reviewed release (14, v0.9.1 source-built offline) plus `legacy_bridge_test`, against catalog-pinned release binaries; cached until their inputs change. Local entrypoint stays `scripts/migration-test/run.sh`. | | `nightly.yml` | Daily schedule, manual | `go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./...` with `BEADS_TEST_SKIP=dolt`; checks coverage >= 30%. | | `nix-build.yml` | Push to `main` paths for Nix or Go module files, manual | `nix build .#default --print-build-logs`; no longer runs on `pull_request` (F7c, spec-f7.md §2.4) since PR Risk's required `test-nix` job (`nix run .#default` plus `nix flake check -L`) is a superset. | diff --git a/engdocs/ICU-POLICY.md b/engdocs/ICU-POLICY.md index 5939860d6..4eac25f12 100644 --- a/engdocs/ICU-POLICY.md +++ b/engdocs/ICU-POLICY.md @@ -57,7 +57,7 @@ Every build path that produces a binary for users must include `-tags gms_pure_g | macOS release | `.github/workflows/release.yml` | | Migration tests | `//tests/migration/...` (Bazel, `bazel.yml` `bazel-test`) | | Nightly tests | `.github/workflows/nightly.yml` | -| Cross-version smoke | `.github/workflows/cross-version-smoke.yml` | +| Cross-version smoke | `//tests/upgrade_smoke/...` (Bazel, `bazel.yml` `bazel-test`) | | Regression tests | `//tests/regression:regression_test` (Bazel, `bazel.yml` `bazel-cmd-dolt`) | ### Canonical pattern: source `.buildflags` diff --git a/internal/httpclient/wire/skew_test.go b/internal/httpclient/wire/skew_test.go index bf431d7c8..04b023171 100644 --- a/internal/httpclient/wire/skew_test.go +++ b/internal/httpclient/wire/skew_test.go @@ -31,7 +31,7 @@ import ( // // The split is the design's (test-lane map, the skew rows): the deterministic, // in-process half runs at PR cadence here, and the released-binary composition -// half runs in cross-version-smoke.yml. Neither substitutes for the other — +// half runs in //tests/upgrade_smoke. Neither substitutes for the other — // this half owns the branches, that half owns the reality check. // // Three skew directions are covered, and they fail in different ways: diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index b4efd8902..443631a45 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -125,6 +125,11 @@ filegroup( visibility = ["//tests/migration:__pkg__"], ) +exports_files( + ["upgrade-smoke-test.sh"], + visibility = ["//tests/upgrade_smoke:__pkg__"], +) + exports_files( [ "ci/fmt-check.sh", diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 100d2e776..517220025 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -1,10 +1,7 @@ package scripts_test import ( - "encoding/json" "fmt" - "os/exec" - "regexp" "sort" "strings" "testing" @@ -13,49 +10,13 @@ import ( ) // F7c (spec-f7.md §2.4, §4.3): advisory workflows moved onto a same-repo-PR -// Blacksmith runner, gained a shared "upgrade-relevant code" path filter, and -// had their matrices folded (Migration Test Harness 14 -> 3, Cross-Version -// Smoke 6 -> 2). These tests pin the invariants that make those changes safe: -// the path filter is identical where it should be, no historical version or -// scenario was dropped by a fold, no advisory job can read a secret just -// because it now names a Blacksmith label, and the Blacksmith-side setup-go -// seed in main.yml actually exists for the jobs that depend on it. - -// advisoryPathFilteredWorkflows are the workflows that share the -// "upgrade-relevant code" allowlist verbatim, save for each one's own -// workflow-file and script entries (spec-f7.md §2.4). conformance.yml and -// migration-test.yml were the others until their tiers moved to Bazel -// (embeddeddolt_conformance_*, //test/conformance:conformance_test, -// //tests/migration). -var advisoryPathFilteredWorkflows = []string{ - "cross-version-smoke.yml", -} - -// advisoryPathFilterBase is the shared prefix of the allowlist: any non-test -// Go change, build input, or embedded schema migration. It must appear, in -// this order, at the start of each of advisoryPathFilteredWorkflows' -// pull_request.paths list. -var advisoryPathFilterBase = []string{ - "**.go", - "!**_test.go", - "go.mod", - "go.sum", - "Makefile", - ".buildflags", - // The .up.sql files are go:embed'd by internal/storage/schema, so they - // change upgrade behavior without matching "**.go"; a migration that - // ships with only a _test.go beside it would otherwise skip all three. - "internal/storage/schema/migrations/**", -} - -// advisoryPathFilterOwnEntries is each workflow's own file/script additions, -// appended after advisoryPathFilterBase. -var advisoryPathFilterOwnEntries = map[string][]string{ - "cross-version-smoke.yml": { - ".github/workflows/cross-version-smoke.yml", - "scripts/upgrade-smoke-test.sh", - }, -} +// Blacksmith runner. These tests pin the invariants that make that safe: no +// advisory job can read a secret just because it now names a Blacksmith +// label, and the Blacksmith-side setup-go seed in main.yml actually exists +// for the jobs that depend on it. The path-filtered upgrade suites F7c folded +// (Migration Test Harness, Cross-Version Smoke) and their filter and fold +// checks are gone: those suites run under Bazel (//tests/migration, +// //tests/upgrade_smoke). type pullRequestPaths struct { On struct { @@ -93,42 +54,6 @@ func readPushPaths(t *testing.T, file string) []string { return parsed.On.Push.Paths } -// TestAdvisoryWorkflowPathFiltersAreIdentical pins that the shared base of -// the "upgrade-relevant code" allowlist is byte-for-byte identical, in the -// same order, across all three workflows it applies to. GitHub Actions has no -// cross-file include for `on:` triggers, so this is the fallback the spec -// explicitly allows: identical literal lists plus a policy test asserting -// they stay identical (spec-f7.md §2.4). -func TestAdvisoryWorkflowPathFiltersAreIdentical(t *testing.T) { - for _, file := range advisoryPathFilteredWorkflows { - paths := readPullRequestPaths(t, file) - if len(paths) < len(advisoryPathFilterBase) { - t.Fatalf("%s pull_request.paths = %v, too short to hold the shared base %v", file, paths, advisoryPathFilterBase) - } - got := paths[:len(advisoryPathFilterBase)] - for i, want := range advisoryPathFilterBase { - if got[i] != want { - t.Errorf("%s pull_request.paths[%d] = %q, want %q (shared base must match byte-for-byte and in order)", file, i, got[i], want) - } - } - } -} - -// TestAdvisoryWorkflowPathFiltersCoverOwnInputs pins that each workflow also -// allowlists its own workflow file and the scripts/fixtures it actually -// exercises, so an edit to e.g. scripts/upgrade-smoke-test.sh is never silently -// skipped by the filter that was added to cut unrelated-PR load. -func TestAdvisoryWorkflowPathFiltersCoverOwnInputs(t *testing.T) { - for file, want := range advisoryPathFilterOwnEntries { - paths := readPullRequestPaths(t, file) - for _, entry := range want { - if !contains(paths, entry) { - t.Errorf("%s pull_request.paths %v does not contain its own entry %q", file, paths, entry) - } - } - } -} - // TestNixBuildDropsPullRequestTriggerNotPushOrDispatch pins the one // "delete the pull_request trigger" trigger change in F7c: nix-build.yml's // PR coverage is fully redundant with PR Risk's required test-nix job (which @@ -199,137 +124,11 @@ func sortedCopy(items []string) []string { return out } -// --- Cross-Version Smoke: 6 -> 2 jobs, chunks of 5, no version dropped ---- - -// ghExtractJQProgram pulls the single-quoted jq program out of a standalone -// `jq - ''` invocation inside a step's `run:` text, so a test -// can execute the REAL program with `jq` directly instead of asserting on a -// substring of the surrounding bash. The flag group is required (one or -// more) specifically so this does not also match `gh`'s own `--jq -// '[.[].tagName]'` filter, which has no `-c`/`-r` flag of its own between -// "jq" and the quoted program. None of this repo's embedded jq programs -// contain a literal single quote, so a non-greedy single-quote match is -// sufficient. -var jqProgramPattern = regexp.MustCompile(`jq(?: -[A-Za-z]+)+ '([^']*)'`) - -func ghExtractJQProgram(t *testing.T, run string) string { - t.Helper() - m := jqProgramPattern.FindStringSubmatch(run) - if m == nil { - t.Fatalf("no `jq '...'` invocation found in:\n%s", run) - } - return m[1] -} - -func runJQ(t *testing.T, program string, stdin string) string { - t.Helper() - requireHostTool(t, "jq") - cmd := exec.Command("jq", "-c", program) - cmd.Stdin = strings.NewReader(stdin) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("jq -c %q <<<%q: %v\n%s", program, stdin, err, out) - } - return strings.TrimSpace(string(out)) -} - -// TestCrossVersionSmokeChunksEveryResolvedVersion runs the REAL jq programs -// embedded in the "Resolve release versions" and "Compute chunk cache key" -// steps against synthetic version lists, instead of asserting on their -// source text (F7c review fix S1, closes mutation M6 `[range(0; length; 5) -// as $i | .[$i:$i+4]]` off-by-one and M7 jq-slice-dropping-versions -// mutations): every resolved version must appear in exactly one chunk, no -// chunk may exceed 5 versions, and the chunk-key step's join must reproduce -// every version in its chunk, in order, space-separated. -func TestCrossVersionSmokeChunksEveryResolvedVersion(t *testing.T) { - workflow := readCIWorkflow(t, "cross-version-smoke.yml") - - versionsJob := workflow.job(t, "versions") - resolve := versionsJob.step(t, "Resolve release versions") - chunkProgram := ghExtractJQProgram(t, resolve.Run) - if versionsJob.Outputs["chunks"] == "" { - t.Errorf("cross-version-smoke.yml's versions job has no chunks output") - } - - for _, n := range []int{0, 1, 4, 5, 6, 9, 10, 29, 30} { - versions := make([]string, 0, n) - for i := 0; i < n; i++ { - versions = append(versions, fmt.Sprintf("v0.%d.0", i)) - } - versionsJSON, err := json.Marshal(versions) - if err != nil { - t.Fatal(err) - } - var chunks [][]string - if err := json.Unmarshal([]byte(runJQ(t, chunkProgram, string(versionsJSON))), &chunks); err != nil { - t.Fatalf("n=%d: chunk output did not parse as [][]string: %v", n, err) - } - wantChunks := (n + 4) / 5 - if n == 0 { - wantChunks = 0 - } - if len(chunks) != wantChunks { - t.Errorf("n=%d: got %d chunks, want %d", n, len(chunks), wantChunks) - } - var flat []string - for _, c := range chunks { - if len(c) > 5 { - t.Errorf("n=%d: chunk %v has more than 5 versions", n, c) - } - flat = append(flat, c...) - } - if !equalStrings(flat, versions) { - t.Errorf("n=%d: concatenated chunks = %v, want exactly the resolved version list %v in order", n, flat, versions) - } - } - - smokeJob := workflow.job(t, "smoke") - if !contains(smokeJob.Needs, "versions") { - t.Errorf("cross-version-smoke.yml's smoke job does not need versions") - } - - chunkKeyStep := smokeJob.step(t, "Compute chunk cache key") - joinProgram := ghExtractJQProgram(t, chunkKeyStep.Run) - for _, chunk := range [][]string{ - {"v1.2.2"}, - {"v1.0.0", "v1.0.1", "v1.1.0", "v1.1.2", "v1.2.2"}, - } { - chunkJSON, err := json.Marshal(chunk) - if err != nil { - t.Fatal(err) - } - got := strings.Trim(runJQ(t, joinProgram, string(chunkJSON)), `"`) - want := strings.Join(chunk, " ") - if got != want { - t.Errorf("chunk-key join(%v) = %q, want %q", chunk, got, want) - } - } - - buildStep := smokeJob.stepIndex(t, "Build candidate binary") - runStep := smokeJob.stepIndex(t, "Run upgrade smoke tests") - if buildStep >= runStep { - t.Errorf("Build candidate binary (index %d) must run before Run upgrade smoke tests (index %d), so the candidate is built once per chunk, not once per version", buildStep, runStep) - } - - run := smokeJob.step(t, "Run upgrade smoke tests") - if run.Env["SMOKE_VERSIONS"] != "${{ steps.chunk.outputs.versions }}" { - t.Errorf("cross-version-smoke.yml's smoke job SMOKE_VERSIONS = %q, want the chunk step's versions output", run.Env["SMOKE_VERSIONS"]) - } - // Exact match (not a substring check, F7c review fix S1): the whole point - // of SMOKE_VERSIONS is that upgrade-smoke-test.sh's own loop consumes it; - // any extra positional arg (e.g. a reintroduced matrix.prev_version) would - // silently make the script test only one version per chunk again. - if want := "./scripts/upgrade-smoke-test.sh"; strings.TrimSpace(run.Run) != want { - t.Errorf("cross-version-smoke.yml's Run upgrade smoke tests run = %q, want exactly %q", run.Run, want) - } -} - // --- Security: no job gains a secret just by naming a Blacksmith label ---- // blacksmithAdvisoryWorkflows are every workflow file F7c moved a job onto a // same-repo-PR (or push-only, for main.yml's seed) Blacksmith runner. var blacksmithAdvisoryWorkflows = []string{ - "cross-version-smoke.yml", "docs-mintlify.yml", "main.yml", } @@ -379,8 +178,6 @@ var advisoryBlacksmithRunnerJobs = []struct { blacksmithLabel string fallback string }{ - {"cross-version-smoke.yml", "smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, - {"cross-version-smoke.yml", "versions", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, {"docs-mintlify.yml", "broken-links", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, } @@ -446,16 +243,13 @@ func TestF7cAdvisorySameRepoBlacksmithExpressionSemantics(t *testing.T) { // blacksmithSetupGoCacheConsumers is every advisory job that restores the // self-defined `blacksmith-sg-v1-` setup-go cache main.yml's // blacksmith-setup-go-cache job seeds (B2, F7c implementation report). -var blacksmithSetupGoCacheConsumers = map[string][]string{ - "cross-version-smoke.yml": {"smoke"}, -} +var blacksmithSetupGoCacheConsumers = map[string][]string{} // blacksmithSetupGoCacheKeyNamespace is the self-defined cache key prefix // (not setup-go's own implicit key) that the seeder and every consumer share, // so the "no save in a consumer job" checks below can scope to exactly this -// cache without also flagging an unrelated, legitimately-caching step (a -// binary cache with a distinct restore/save pair of its own - see -// advisoryBinaryCaches below). +// cache without also flagging an unrelated, legitimately-caching step in +// another namespace. const blacksmithSetupGoCacheKeyNamespace = "blacksmith-sg-v1-" // TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers pins that main.yml's @@ -636,10 +430,10 @@ func TestMainWorkflowHasNoSameRepoPRReachableTrigger(t *testing.T) { // main.yml's seeder may ever write it, so a same-repo PR run can read the // cache but never poison what another PR or main's seeder reads back. The // "no save" check is scoped to the blacksmith-sg-v1- key namespace -// specifically (not "no actions/cache/save in this job at all"), since a -// binary cache in a different namespace is governed by its own -// restore/save-gated pair below (advisoryBinaryCaches / F7c review fix X1), -// not an exemption from this one. +// specifically (not "no actions/cache/save in this job at all"): a cache in +// another namespace is the general sweep's concern below +// (TestBlacksmithReachableAdvisoryJobsNeverSaveACache), not an exemption +// from this one. func TestAdvisoryBlacksmithConsumersAreCacheRestoreOnly(t *testing.T) { for file, jobNames := range blacksmithSetupGoCacheConsumers { workflow := readCIWorkflow(t, file) @@ -757,81 +551,6 @@ func TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers(t *testing.T) { } } -// --- Binary caches: restore-always, save only off pull_request ------------ - -// advisoryBinaryCaches are the per-binary caches F7c review fixes B2 and X1 -// converted from a monolithic (auto-saving) actions/cache into an explicit -// restore/save pair, so a same-repo PR can read a previously published -// binary but never publish its own into a cache another run would trust -// unverified. migration-test.yml's historical-dolt-* cache was originally -// exempted here on the theory that scripts/migration-test/lib/binary.sh's -// sha256 verification of every extracted archive made a bare, auto-saving -// actions/cache safe regardless of who wrote it. That exemption was unsound -// (F7c review fix X1): `tar -P` extraction plus lib/binary.sh's `cp -f` -// following a symlink planted inside the archive can redirect the final copy -// to an arbitrary path (e.g. over the checked-out workspace or the candidate -// binary itself) before the checksum check ever runs, so checksum -// verification alone does not make a same-repo-PR-writable cache entry safe -// to trust. historical-dolt-* now gets the same restore-always/save-off-PR -// split as the other two. -var advisoryBinaryCaches = []struct { - file, job, restoreStep, saveStep, keyPrefix string - // wantSaveIf is the save step's `if:` condition, required byte-for-byte - // (F7c review fix S3): a `strings.Contains` check here would pass under - // e.g. `... || true`, which always evaluates true and silently - // reintroduces the same-repo-PR poisoning path this whole table exists - // to close. - wantSaveIf string -}{ - {"cross-version-smoke.yml", "smoke", "Restore previous release binaries cache", "Save previous release binaries cache", "smoke-binaries-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"}, -} - -// TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated pins the restore/save -// split itself (F7c review fix B2): the restore step always runs (modulo the -// job's own pre-existing gate, if any), -// the save step additionally requires an exact allow-list of -// `github.event_name == 'push' || github.event_name == 'workflow_dispatch'` -// (F7c review fix, discovered via the S2 sweep: a deny-list of -// `!= 'pull_request'` also admits a hypothetical future merge_group event, -// which each of these three jobs' runs-on expressions already resolve to -// Blacksmith for), both steps key off the same cache, and no monolithic -// (bare) actions/cache step remains for either binary cache - a monolithic -// step would silently reintroduce the auto-save-on-any-PR poisoning path B2 -// closes. -func TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated(t *testing.T) { - for _, c := range advisoryBinaryCaches { - t.Run(c.file, func(t *testing.T) { - workflow := readCIWorkflow(t, c.file) - job := workflow.job(t, c.job) - - restore := job.step(t, c.restoreStep) - if actionFamily(restore.Uses) != cacheRestoreActionFamily { - t.Errorf("%s job %s step %q uses %q, want family %q", c.file, c.job, c.restoreStep, restore.Uses, cacheRestoreActionFamily) - } - if !strings.HasPrefix(restore.With["key"], c.keyPrefix) { - t.Errorf("%s job %s step %q key = %q, want prefix %q", c.file, c.job, c.restoreStep, restore.With["key"], c.keyPrefix) - } - - save := job.step(t, c.saveStep) - if actionFamily(save.Uses) != cacheSaveActionFamily { - t.Errorf("%s job %s step %q uses %q, want family %q", c.file, c.job, c.saveStep, save.Uses, cacheSaveActionFamily) - } - if save.If != c.wantSaveIf { - t.Errorf("%s job %s step %q has if=%q, want exactly %q", c.file, c.job, c.saveStep, save.If, c.wantSaveIf) - } - if save.With["key"] != restore.With["key"] { - t.Errorf("%s job %s: restore key %q != save key %q", c.file, c.job, restore.With["key"], save.With["key"]) - } - - for _, step := range job.Steps { - if actionFamily(step.Uses) == cacheMonolithicActionFamily && strings.HasPrefix(step.With["key"], c.keyPrefix) { - t.Errorf("%s job %s has a monolithic actions/cache step keyed %q; B2 requires an explicit restore/save split here", c.file, c.job, step.With["key"]) - } - } - }) - } -} - // --- General sweep: no Blacksmith-reachable advisory job may save a cache -- // generalCacheSweepWorkflows is every F7c advisory workflow plus main.yml diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index b3f0980c6..ae88cefb7 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -126,7 +126,7 @@ func TestPullRequestWorkflowsTriggerOnHotfixBranches(t *testing.T) { } root := sourceRepoRoot(t) for _, name := range []string{ - "pr.yml", prRiskWorkflowName, "cross-version-smoke.yml", + "pr.yml", prRiskWorkflowName, } { t.Run(name, func(t *testing.T) { var doc triggers @@ -2709,9 +2709,8 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && // F7c: used to define its own sameRepoBlacksmith4vcpu here (same expression // as sameRepoBlacksmith2vcpu with the 4 vCPU label, for advisory jobs that -// compile Go: cross-version-smoke.yml). F7a independently defined the same -// const; both are now served by the single shared sameRepoBlacksmith4vcpu in -// ci_blacksmith_runner_test.go. +// compile Go). F7a independently defined the same const; both are now served +// by the single shared sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go. // F7b: venue-matrix runs-on for main.yml jobs that seed a Blacksmith GOCACHE // alongside their GitHub-hosted seed ("venue: [blacksmith, github]"): @@ -3038,8 +3037,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // is cheap enough to size at 2 vCPU per spec-f7.md §2.2) and moves // to Blacksmith for same-repo PRs/merge_group only; forks, // Dependabot and push stay on ubuntu-latest. - "cross-version-smoke.yml": {"smoke": sameRepoBlacksmith4vcpu, "versions": sameRepoBlacksmith2vcpu}, - "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, + "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, // main.yml's seeder job (B2, F7c implementation report) is the one // Blacksmith job that is NOT gated by the same-repo-PR expression: it // is push-to-main only (always trusted), so it wraps the literal @@ -3103,7 +3101,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // Blacksmith label beyond the jobs listed in `want` above. for _, file := range []string{ "pr.yml", "pr-risk.yml", "main.yml", - "cross-version-smoke.yml", "docs-mintlify.yml", + "docs-mintlify.yml", } { workflow := readCIWorkflow(t, file) allowed := want[file] diff --git a/scripts/upgrade-smoke-test.sh b/scripts/upgrade-smoke-test.sh index d5d426b4f..52b05fbf2 100755 --- a/scripts/upgrade-smoke-test.sh +++ b/scripts/upgrade-smoke-test.sh @@ -103,6 +103,17 @@ case "$ARCH" in esac get_previous_binary() { + # PREV_BIN (set by //tests/upgrade_smoke's targets) is the previous + # release already pinned and sha256-verified against the release catalog, + # so nothing is downloaded; it must report PREV_VERSION. + if [ -n "${PREV_BIN:-}" ]; then + if ! "$PREV_BIN" version 2>/dev/null | grep -qF " ${PREV_VER_BARE} "; then + echo -e "${RED}PREV_BIN=${PREV_BIN} does not report ${PREV_VERSION}${NC}" >&2 + exit 1 + fi + echo "$PREV_BIN" + return + fi local cached="$CACHE_DIR/bd-${PREV_VER_BARE}" if [ -x "$cached" ]; then echo "$cached" diff --git a/tests/upgrade_smoke/BUILD.bazel b/tests/upgrade_smoke/BUILD.bazel new file mode 100644 index 000000000..be70457de --- /dev/null +++ b/tests/upgrade_smoke/BUILD.bazel @@ -0,0 +1,43 @@ +load(":defs.bzl", "upgrade_smoke_tests") + +# The upgrade smoke tests (formerly cross-version-smoke.yml): one target per +# previous release, upgrade_smoke__test, running +# scripts/upgrade-smoke-test.sh from the pinned release +# (@bd_releases, sha256-verified against the release catalog) to the +# non-race bd_for_tests. The releases are the newest `window` the catalog +# pins a linux/amd64 asset for (its stable GitHub releases), so a release +# added to the catalog joins the window and the oldest drops out. The PR-core +# lane (bazel-test) runs them remotely on every PR, merge group and push, and +# a target re-runs only when the candidate, the script or its release +# changes. +# +# Tags: +# pr-core-only: shell tests against the release-shaped bd_for_tests (no +# integration tag); the integration lane, which requires Go testcases, +# leaves them out. +upgrade_smoke_tests( + tags = ["pr-core-only"], + window = 30, +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tests/upgrade_smoke/defs.bzl b/tests/upgrade_smoke/defs.bzl new file mode 100644 index 000000000..3c3045e56 --- /dev/null +++ b/tests/upgrade_smoke/defs.bzl @@ -0,0 +1,32 @@ +"""One upgrade smoke test per pinned previous release (see BUILD.bazel).""" + +load("@bd_releases//:versions.bzl", "VERSIONS") +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") + +def upgrade_smoke_tests(window, tags): + """Declares upgrade_smoke__test for the newest `window` catalog releases. + + Args: + window: how many of the newest pinned releases to test. + tags: the tests' tags. + """ + for v in VERSIONS[-window:]: + sh_test( + name = "upgrade_smoke_{}_test".format(v.replace(".", "_")), + size = "large", + srcs = ["upgrade_smoke_test.sh"], + args = [v], + data = [ + "//:.buildflags", + "//cmd/bd:bd_for_tests", + "//scripts:upgrade-smoke-test.sh", + "@bd_releases//:" + v, + ], + env = { + "CANDIDATE_BIN": "$(rootpath //cmd/bd:bd_for_tests)", + "PREV_BIN": "$(rootpath @bd_releases//:{})".format(v), + }, + tags = tags, + target_compatible_with = LINUX_AMD64, + ) diff --git a/tests/upgrade_smoke/upgrade_smoke_test.sh b/tests/upgrade_smoke/upgrade_smoke_test.sh new file mode 100755 index 000000000..24130e901 --- /dev/null +++ b/tests/upgrade_smoke/upgrade_smoke_test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# One previous release of the upgrade smoke tests (formerly +# cross-version-smoke.yml's `scripts/upgrade-smoke-test.sh` loop): that +# release → the candidate, with both binaries pinned inputs, so the script +# downloads and builds nothing. +# +# The first argument is the release. CANDIDATE_BIN and PREV_BIN are +# runfiles-relative paths ($(rootpath ...), relative to the working directory +# Bazel starts the test in), made absolute here before the script changes +# directory. Any further arguments (a lane's --test_arg flags, meant for Go +# test binaries) are ignored. +set -euo pipefail +release="$1" +CANDIDATE_BIN="$PWD/${CANDIDATE_BIN:?}" +PREV_BIN="$PWD/${PREV_BIN:?}" +export CANDIDATE_BIN PREV_BIN +exec scripts/upgrade-smoke-test.sh "$release"