diff --git a/.bazelrc b/.bazelrc index 705245c2b..e298cc59b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -216,7 +216,9 @@ test:doltserver-integration --experimental_remote_cache_eviction_retries=0 # tag), the whole integration-tagged cmd/bd suite against a hermetic dolt # sql-server, i.e. the Dolt-gated cmd/bd tests every other lane skips # (BEADS_TEST_SKIP=dolt in --config=prcore/ci and --config=integration, and -# in nightly.yml's go test job). The build flags are exactly +# in nightly.yml's go test job), and //tests/regression:regression_test +# (the differential regression suite, formerly regression.yml, against the +# catalog-pinned baseline in @bd_releases). The build flags are exactly # --config=integration's, so it compiles nothing bazel-integration does not # (shared through the remote cache). bazel.yml runs it where it runs the # integration lane (remote, or with the read-only cache in mode cache), in a diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 07355f39c..578385275 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1723,9 +1723,12 @@ jobs: # shards) against a hermetic dolt sql-server, i.e. the Dolt-gated cmd/bd # tests (TestCLI_*, TestInit*, the store-backed suites) that every other # lane and legacy job skips with BEADS_TEST_SKIP=dolt or leaves out of its - # manifest. --config=doltserver-cmd has exactly --config=integration's - # build flags, so it shares bazel-integration's compile through the - # remote cache. Runs exactly where bazel-integration runs (the same if): + # manifest; plus the differential regression suite + # (//tests/regression:regression_test, 8 shards: the candidate + # bd_for_tests against the catalog-pinned v0.49.6 release, formerly + # regression.yml). --config=doltserver-cmd has exactly + # --config=integration's build flags, so it shares bazel-integration's + # compile through the remote cache. Runs exactly where bazel-integration runs (the same if): # remotely (modes remote, fork-ro, fork-rw), or locally with the # read-only cache (mode cache: fork and Dependabot PRs while rbe-fork is # closed), never plain local (rbe=off), and not when a caller passes diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0b5cff73..2ad147015 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1163,9 +1163,6 @@ jobs: - name: Build bd run: make build - - name: Compile regression tests - run: go test -c -tags regression,gms_pure_go ./tests/regression - # Always save (even if a warm-up step above failed) under a fresh, # commit-specific key: the content is still a valid, newer GOCACHE/ # GOMODCACHE snapshot than whatever was restored, and `cache/save` is a diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml deleted file mode 100644 index 52c03931f..000000000 --- a/.github/workflows/regression.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: Regression Tests - -on: - # Run on merges to main - push: - branches: [main] - pull_request: - branches: [main, 'release/**', 'hotfix/**'] - # Allow manual trigger - workflow_dispatch: - -env: - BD_DISABLE_METRICS: "1" - BD_DISABLE_EVENT_FLUSH: "1" - -jobs: - regression: - name: Differential Regression (v0.49.6 baseline) - # F7c: detect-regression-need folded in as this job's own first step - # (id: detect). A non-applicable PR now costs one short Blacksmith-sized - # job instead of two GitHub-hosted jobs; every later step is gated on - # steps.detect.outputs.run_regression instead of a separate job's output. - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 45 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Decide whether to run differential regression tests - id: detect - env: - EVENT_NAME: ${{ github.event_name }} - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - set -euo pipefail - - if [[ "$EVENT_NAME" != "pull_request" ]]; then - echo "run_regression=true" >> "$GITHUB_OUTPUT" - echo "reason=$EVENT_NAME event always runs regression tests" >> "$GITHUB_OUTPUT" - exit 0 - fi - - labels="$(jq -r '.pull_request.labels[].name // empty' "$GITHUB_EVENT_PATH")" - if grep -Fxq "run-regression" <<<"$labels"; then - echo "run_regression=true" >> "$GITHUB_OUTPUT" - echo "reason=run-regression label present" >> "$GITHUB_OUTPUT" - exit 0 - fi - if grep -Fxq "skip-regression" <<<"$labels"; then - echo "run_regression=false" >> "$GITHUB_OUTPUT" - echo "reason=skip-regression label present" >> "$GITHUB_OUTPUT" - exit 0 - fi - - changed="$(git diff --name-only "$PR_BASE_SHA" "$PR_HEAD_SHA")" - echo "Changed files:" - printf '%s\n' "$changed" - - regression_candidates="$( - printf '%s\n' "$changed" | - grep -Ev '^(cmd/bd|internal/storage|internal/types)/.*_test\.go$' || - true - )" - - risky_pattern='^(\.github/workflows/regression\.yml|Makefile|go\.(mod|sum)|cmd/bd/|internal/storage/|internal/types/|tests/regression/)' - if grep -Eq "$risky_pattern" <<<"$regression_candidates"; then - echo "run_regression=true" >> "$GITHUB_OUTPUT" - echo "reason=PR changes CLI, storage, types, regression, or build inputs" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "run_regression=false" >> "$GITHUB_OUTPUT" - echo "reason=PR does not touch regression-sensitive paths" >> "$GITHUB_OUTPUT" - - - name: Show regression trigger reason - if: steps.detect.outputs.run_regression == 'true' - run: echo "${{ steps.detect.outputs.reason }}" - - - name: Set up Go - id: setup-go - if: steps.detect.outputs.run_regression == 'true' - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: ${{ runner.environment == 'github-hosted' }} - - # F7c review fix S5: this job's own key copy must match main.yml's - # seeder key byte-for-byte (TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers), - # which now keys by UTC calendar day instead of github.sha so main - # pushes don't churn a multi-GB cache entry on every commit; this step - # computes that same day value locally so the key expression below - # resolves (steps.* references only resolve within the same job). - - name: Compute cache date - id: cache-date - if: steps.detect.outputs.run_regression == 'true' && runner.environment == 'self-hosted' - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Blacksmith setup-go cache - if: steps.detect.outputs.run_regression == 'true' && runner.environment == 'self-hosted' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: | - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}- - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Configure Git - if: steps.detect.outputs.run_regression == 'true' - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Install Dolt - if: steps.detect.outputs.run_regression == 'true' - run: ./scripts/ci/install-dolt.sh - - - name: Configure Dolt - if: steps.detect.outputs.run_regression == 'true' - run: | - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - dolt version - - - name: Pull Dolt sql-server image - # Without this, isDoltImageCached() returns false in TestMain and the - # regression suite falls back to embedded Dolt with HOME=, which - # has a known cleanup race (see beads issue: TestCreateWithParentFlag - # TempDir RemoveAll "directory not empty" flake). Keep the tag in sync - # with internal/testutil/testdoltcommon.go:DoltDockerImage. - if: steps.detect.outputs.run_regression == 'true' - run: ./scripts/ci/pull-dolt-image.sh - - # Restore-only: actions/cache (bare) auto-saves on any key miss, which - # would let a same-repo PR that bumps BASELINE_VERSION write a baseline - # binary cache entry a later PR or main could read back unverified. The - # matching save below only runs off pull_request (B2, F7c implementation - # report). - - name: Restore baseline binary cache - if: steps.detect.outputs.run_regression == 'true' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: regression-baseline-${{ hashFiles('tests/regression/BASELINE_VERSION') }} - - # Passing-side wall time already overlapped the go deadline below, so - # ordinary GitHub-runner speed variance -- not a real hang -- was - # tripping this step. Measured over the 100 most recent Regression - # Tests runs (2026-09-03/04, ~29.5h): p50 593s (~10m), max passing - # 1189s (~19.8m) against the prior 1200s (20m) cap, and 5 runs in that - # window hit the identical `panic: test timed out after 20m0s` with - # zero `--- FAIL` lines -- a budget defect, not a regression. Match - # main.yml's Test job fix for the same failure mode: raise the go - # deadline to cover the observed slowdown and keep it inside a larger - # finite job budget (see timeout-minutes above), so the go deadline - # fires first and prints goroutine stacks -- a REAL hang stays - # diagnosable instead of being cut off by a bare job kill. - - name: Run regression tests - if: steps.detect.outputs.run_regression == 'true' - run: go test -tags=regression,gms_pure_go -timeout=40m -v ./tests/regression/... - - - name: Save baseline binary cache - # Allow-list push/workflow_dispatch rather than deny-list - # pull_request: a deny-list also admits a hypothetical future - # merge_group event, which this job's runs-on expression already - # resolves to Blacksmith for. Pinned by - # TestBlacksmithReachableAdvisoryJobsNeverSaveACache (checks both the - # pull_request and merge_group trust contexts independently). - if: steps.detect.outputs.run_regression == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: regression-baseline-${{ hashFiles('tests/regression/BASELINE_VERSION') }} diff --git a/BUILD.bazel b/BUILD.bazel index dd3c4b4a1..5ece252ab 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -367,6 +367,7 @@ filegroup( "//test/conformance:repo_files", "//test/docsync:repo_files", "//test/testmainconvention:repo_files", + "//tests/regression:repo_files", "//third_party/cc_toolchain:repo_files", "//third_party/patches:repo_files", "//tools/bazel:repo_files", diff --git a/MODULE.bazel b/MODULE.bazel index 186368330..15033473b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -234,3 +234,11 @@ new_local_repository( """, path = ".beads/formulas", ) + +# Historical bd releases for the regression, migration and cross-version +# suites (tools/bazel/bd_releases.bzl): one repository per release in the +# release catalog that records a linux_amd64 asset digest, behind the +# @bd_releases hub. Adding a release is a catalog change only. +bd_releases = use_extension("//tools/bazel:bd_releases.bzl", "bd_releases") +bd_releases.catalog(path = "//scripts:migration-test/release-catalog.json") +use_repo(bd_releases, "bd_releases") diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 9c4e304f5..c7f630716 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -495,6 +495,7 @@ go_binary( "//internal/beads:__pkg__", "//internal/storage/uow:__pkg__", "//test/conformance:__pkg__", + "//tests/regression:__pkg__", ], ) diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 6ec63d432..1c03ab3fa 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -151,10 +151,12 @@ Current PR-related workflow names: Runs on pushes to `main`. Contains the main branch health checks, package gates, platform smoke/short coverage, embedded Dolt coverage, and promoted Linux no-short integration shards. -- `.github/workflows/regression.yml`: `Regression Tests` - Runs on `pull_request`, `push` to `main`, and manual dispatch. Does not - currently run on `merge_group`. Uses job-level conditional regression - execution. +- The differential regression suite (`tests/regression`, formerly + `regression.yml`) is `//tests/regression:regression_test`, tagged + `dolt-server-cmd`, so `bazel.yml`'s `bazel-cmd-dolt` lane runs it on every + PR, merge group and push to `main` (results cached until its inputs + change), against the catalog-pinned `@bd_releases//:v0.49.6` baseline and + a hermetic dolt sql-server. - `.github/workflows/cross-version-smoke.yml`: `Cross-Version Smoke Tests` Runs on every PR to `main`, tag pushes, and manual dispatch. Does not currently run on `merge_group`. @@ -337,8 +339,7 @@ PR. It runs in the default branch's cache scope, which `push` runs on binaries); - the `beads-go-build-v2-*` GOCACHE entries restored by `main.yml` and `pr.yml`; - - the executables in `smoke-binaries-*`, `historical-dolt-*` and - `regression-baseline-*`. + - the executables in `smoke-binaries-*` and `historical-dolt-*`. GitHub's read-only default covers all of them. Hardening `release.yml` with `cache: false` is tracked separately. @@ -661,7 +662,6 @@ Do not require these existing check names directly: - `Test (Windows - smoke)` - `PR Lint (native)`, `PR Lint (windows)` and `PR Lint (darwin)` - `Test Nix Flake` -- `Differential Regression (v0.49.6 baseline)` - `Upgrade smoke (chunk N)` (F7c: folded from one job per version into one job per 5-version chunk; still never require a matrix-expanded chunk job directly) @@ -1316,20 +1316,16 @@ tier down in the same workflow: ### Regression Tests -`Regression Tests` can stay visible as a non-required workflow. If regression -becomes branch-protection relevant, do not require -`Differential Regression (v0.49.6 baseline)` directly. - -Use one of these narrow changes instead: - -1. Move the regression detector and regression job into the required PR - topology, wire them into the relevant aggregate gate, and add `merge_group` - behavior that defaults to running regression. -2. Keep `regression.yml` separate, remove any workflow-level skip filters, add - `merge_group`, add a final `Regression Gate / Informational` aggregate, and - leave it non-required unless branch protection is intentionally expanded. - -The preferred required-check topology keeps only aggregate gates required. +`regression.yml` is retired. The suite runs as +`//tests/regression:regression_test` (8 shards) in `bazel.yml`'s +`bazel-cmd-dolt` lane, which `pr.yml`'s `CI Gate / Required` requires +(`BAZEL_CMD_DOLT_REQUIRED`), so regression is now part of the aggregate gate +on PRs and merge groups instead of a separate, path-detected advisory +workflow. Its path detector is replaced by test result caching: the target +re-runs only when `bd_for_tests`, the suite or its pinned inputs change. The +baseline is `@bd_releases//:` (`tools/bazel/bd_releases.bzl`, +pinned by `scripts/migration-test/release-catalog.json`); `TestMain` fails if +the binary does not report `BASELINE_VERSION`. ### Nix Build diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index fee86d199..5936cf71e 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -199,7 +199,7 @@ Key jobs preserved by display name: | Workflow | Triggers | Main validation | |---|---|---| -| `regression.yml` | Push to `main`, PR to `main`, manual | Detector runs regression on push/manual, PR label `run-regression`, or risky paths; test command is `go test -tags=regression,gms_pure_go -timeout=20m -v ./tests/regression/...`. | +| `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (8 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | | `cross-version-smoke.yml` | Tags, PRs, manual | PRs test latest 5 releases, tags test latest 30, via `scripts/upgrade-smoke-test.sh`. | | `migration-test.yml` | Tags, PRs touching upgrade-relevant code (advisory, path-filtered), manual | Builds candidate once per shard (3 shards, folded from 14 per-version legs, F7c) and loops `scripts/migration-test/run.sh --version` over each shard's versions; not a required PR/main gate. | | `nightly.yml` | Daily schedule, manual | `go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./...` with `BEADS_TEST_SKIP=dolt`; checks coverage >= 30%. | diff --git a/engdocs/ICU-POLICY.md b/engdocs/ICU-POLICY.md index 487541077..a8ef447f4 100644 --- a/engdocs/ICU-POLICY.md +++ b/engdocs/ICU-POLICY.md @@ -58,7 +58,7 @@ Every build path that produces a binary for users must include `-tags gms_pure_g | Migration tests | `.github/workflows/migration-test.yml` | | Nightly tests | `.github/workflows/nightly.yml` | | Cross-version smoke | `.github/workflows/cross-version-smoke.yml` | -| Regression tests | `.github/workflows/regression.yml` | +| Regression tests | `//tests/regression:regression_test` (Bazel, `bazel.yml` `bazel-cmd-dolt`) | ### Canonical pattern: source `.buildflags` diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 8b20298fa..5f193ae29 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -694,7 +694,6 @@ func TestCrossVersionSmokeChunksEveryResolvedVersion(t *testing.T) { // blacksmithAdvisoryWorkflows are every workflow file F7c moved a job onto a // same-repo-PR (or push-only, for main.yml's seed) Blacksmith runner. var blacksmithAdvisoryWorkflows = []string{ - "regression.yml", "migration-test.yml", "cross-version-smoke.yml", "docs-mintlify.yml", @@ -750,7 +749,6 @@ var advisoryBlacksmithRunnerJobs = []struct { blacksmithLabel string fallback string }{ - {"regression.yml", "regression", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, {"migration-test.yml", "historical-upgrades", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-24.04"}, {"cross-version-smoke.yml", "smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, {"cross-version-smoke.yml", "versions", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, @@ -820,7 +818,6 @@ func TestF7cAdvisorySameRepoBlacksmithExpressionSemantics(t *testing.T) { // self-defined `blacksmith-sg-v1-` setup-go cache main.yml's // blacksmith-setup-go-cache job seeds (B2, F7c implementation report). var blacksmithSetupGoCacheConsumers = map[string][]string{ - "regression.yml": {"regression"}, "migration-test.yml": {"historical-upgrades"}, "cross-version-smoke.yml": {"smoke"}, } @@ -886,7 +883,6 @@ func TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers(t *testing.T) { wantWarmups := []string{ "make build", - "go test -c -tags regression,gms_pure_go ./tests/regression", } for _, want := range wantWarmups { found := false @@ -1161,13 +1157,12 @@ var advisoryBinaryCaches = []struct { wantSaveIf string }{ {"cross-version-smoke.yml", "smoke", "Restore previous release binaries cache", "Save previous release binaries cache", "smoke-binaries-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"}, - {"regression.yml", "regression", "Restore baseline binary cache", "Save baseline binary cache", "regression-baseline-", "steps.detect.outputs.run_regression == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')"}, {"migration-test.yml", "historical-upgrades", "Restore pinned historical release cache", "Save pinned historical release cache", "historical-dolt-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"}, } // TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated pins the restore/save // split itself (F7c review fix B2): the restore step always runs (modulo the -// job's own pre-existing gate, e.g. regression's run_regression detection), +// job's own pre-existing gate, if any), // the save step additionally requires an exact allow-list of // `github.event_name == 'push' || github.event_name == 'workflow_dispatch'` // (F7c review fix, discovered via the S2 sweep: a deny-list of @@ -1211,178 +1206,6 @@ func TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated(t *testing.T) { } } -// TestRegressionStepsAfterDetectAreGated pins that every step after -// regression.yml's "Decide whether to run differential regression tests" -// (id: detect) stays gated on steps.detect.outputs.run_regression == 'true' -// (F7c review fix S1, closes an "ungated regression step" mutation): the -// fold that moved detect-regression-need into this job's own first step -// means a single dropped `if:` would make an inapplicable PR silently pay -// for (or worse, run) the rest of the job instead of skipping it. -func TestRegressionStepsAfterDetectAreGated(t *testing.T) { - job := readCIWorkflow(t, "regression.yml").job(t, "regression") - detectIndex := job.stepIndex(t, "Decide whether to run differential regression tests") - const want = "steps.detect.outputs.run_regression == 'true'" - for _, step := range job.Steps[detectIndex+1:] { - if !strings.Contains(step.If, want) { - t.Errorf("regression.yml step %q has if=%q, want it to contain %q", step.Name, step.If, want) - } - } -} - -// TestRegressionDetectStepBehavioral runs regression.yml's "Decide whether to -// run differential regression tests" (id: detect) step for real, the same -// way TestMigrationHarnessLoopExecutesBehaviorally exercises the migration -// loop (F7c review fix S1, closes mutation M13: a detect step hardcoded to -// always emit run_regression=false). TestRegressionStepsAfterDetectAreGated -// only pins that steps AFTER detect stay gated on its output; it says nothing -// about whether detect's own logic ever produces "true", so a detect step -// that always reports false would make every downstream gate vacuously -// satisfied while regression silently never ran. This executes the step's -// actual bash against real GITHUB_OUTPUT/GITHUB_EVENT_PATH files and a real -// two-commit git repo for each input that must change the verdict. -func TestRegressionDetectStepBehavioral(t *testing.T) { - requireHostTool(t, "bash") - requireHostTool(t, "jq") - requireHostTool(t, "git") - - job := readCIWorkflow(t, "regression.yml").job(t, "regression") - step := job.step(t, "Decide whether to run differential regression tests") - if step.ID != "detect" { - t.Fatalf("detect step id = %q, want %q", step.ID, "detect") - } - - // newRepo creates a base commit, then (if any changedFiles are given) a - // second commit that adds each of them, and returns the repo dir plus - // both commit SHAs for PR_BASE_SHA/PR_HEAD_SHA. - newRepo := func(t *testing.T, changedFiles ...string) (dir, base, head string) { - t.Helper() - dir = t.TempDir() - git := func(args ...string) string { - t.Helper() - cmd := exec.Command("git", args...) - cmd.Dir = dir - cmd.Env = append(os.Environ(), - "GIT_CONFIG_NOSYSTEM=1", - "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t.test", - "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t.test", - ) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - return strings.TrimSpace(string(out)) - } - git("init", "-q", "-b", "main") - if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte("base\n"), 0o644); err != nil { - t.Fatal(err) - } - git("add", "README.md") - git("commit", "-q", "-m", "base") - base = git("rev-parse", "HEAD") - - for _, f := range changedFiles { - full := filepath.Join(dir, f) - if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(full, []byte("x\n"), 0o644); err != nil { - t.Fatal(err) - } - git("add", f) - } - if len(changedFiles) > 0 { - git("commit", "-q", "-m", "head") - } - head = git("rev-parse", "HEAD") - return dir, base, head - } - - writeEvent := func(t *testing.T, dir string, labels ...string) string { - t.Helper() - type label struct { - Name string `json:"name"` - } - type prBody struct { - Labels []label `json:"labels"` - } - type event struct { - PullRequest prBody `json:"pull_request"` - } - e := event{PullRequest: prBody{Labels: []label{}}} - for _, l := range labels { - e.PullRequest.Labels = append(e.PullRequest.Labels, label{Name: l}) - } - b, err := json.Marshal(e) - if err != nil { - t.Fatal(err) - } - p := filepath.Join(dir, "event.json") - if err := os.WriteFile(p, b, 0o644); err != nil { - t.Fatal(err) - } - return p - } - - runDetect := func(t *testing.T, dir, eventPath, eventName, base, head string) (runRegression, reason string) { - t.Helper() - outFile := filepath.Join(t.TempDir(), "output") - if err := os.WriteFile(outFile, nil, 0o644); err != nil { - t.Fatal(err) - } - cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step.Run) - cmd.Dir = dir - cmd.Env = append(os.Environ(), - "EVENT_NAME="+eventName, - "PR_BASE_SHA="+base, - "PR_HEAD_SHA="+head, - "GITHUB_OUTPUT="+outFile, - "GITHUB_EVENT_PATH="+eventPath, - ) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("detect step: %v\noutput:\n%s", err, out) - } - outBytes, err := os.ReadFile(outFile) - if err != nil { - t.Fatal(err) - } - for _, line := range strings.Split(string(outBytes), "\n") { - if v, ok := strings.CutPrefix(line, "run_regression="); ok { - runRegression = v - } - if v, ok := strings.CutPrefix(line, "reason="); ok { - reason = v - } - } - return runRegression, reason - } - - cases := []struct { - name string - eventName string - labels []string - changedFiles []string - want string - }{ - {"push always runs", "push", nil, nil, "true"}, - {"run-regression label forces true", "pull_request", []string{"run-regression"}, []string{"docs/unrelated.md"}, "true"}, - {"skip-regression label forces false", "pull_request", []string{"skip-regression"}, []string{"cmd/bd/x.go"}, "false"}, - {"cmd/bd non-test change runs", "pull_request", nil, []string{"cmd/bd/x.go"}, "true"}, - {"only a cmd/bd test file changed skips", "pull_request", nil, []string{"cmd/bd/x_test.go"}, "false"}, - {"docs-only change skips", "pull_request", nil, []string{"docs/a.md"}, "false"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - dir, base, head := newRepo(t, c.changedFiles...) - eventPath := writeEvent(t, dir, c.labels...) - got, reason := runDetect(t, dir, eventPath, c.eventName, base, head) - if got != c.want { - t.Errorf("run_regression = %q (reason %q), want %q", got, reason, c.want) - } - }) - } -} - // --- General sweep: no Blacksmith-reachable advisory job may save a cache -- // generalCacheSweepWorkflows is every F7c advisory workflow plus main.yml diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index fcd7006e2..7145eb2e5 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -110,7 +110,7 @@ func TestPullRequestWorkflowsTriggerOnHotfixBranches(t *testing.T) { } root := sourceRepoRoot(t) for _, name := range []string{ - "pr.yml", prRiskWorkflowName, "cross-version-smoke.yml", "regression.yml", + "pr.yml", prRiskWorkflowName, "cross-version-smoke.yml", } { t.Run(name, func(t *testing.T) { var doc triggers @@ -1113,7 +1113,7 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { // internal/testutil gates on `docker image inspect` and never auto-pulls. The // other jobs that do pull it are pr.yml/contract-corpus, // pr-risk.yml/test-proxied-cmd, -// pr-risk.yml/test-server-storage and -full, regression.yml/regression, and +// pr-risk.yml/test-server-storage and -full, and // bazel.yml's --config=docker lane. They are out of scope for this change, not // immune: no reap has been attributed to them, and the docker lane would // additionally need --test_env=TESTCONTAINERS_RYUK_DISABLED=true because bazel @@ -2689,7 +2689,7 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && // F7c: used to define its own sameRepoBlacksmith4vcpu here (same expression // as sameRepoBlacksmith2vcpu with the 4 vCPU label, for advisory jobs that -// compile Go: regression.yml, migration-test.yml, +// compile Go: migration-test.yml, // cross-version-smoke.yml). F7a independently // defined the same const; both are now served by the single shared // sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go. migration-test.yml's @@ -3032,7 +3032,6 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // is cheap enough to size at 2 vCPU per spec-f7.md ยง2.2) and moves // to Blacksmith for same-repo PRs/merge_group only; forks, // Dependabot and push stay on ubuntu-latest. - "regression.yml": {"regression": sameRepoBlacksmith4vcpu}, "migration-test.yml": {"historical-upgrades": sameRepoBlacksmith4vcpuNoble}, "cross-version-smoke.yml": {"smoke": sameRepoBlacksmith4vcpu, "versions": sameRepoBlacksmith2vcpu}, "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, @@ -3099,7 +3098,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // Blacksmith label beyond the jobs listed in `want` above. for _, file := range []string{ "pr.yml", "pr-risk.yml", "main.yml", - "regression.yml", "migration-test.yml", + "migration-test.yml", "cross-version-smoke.yml", "docs-mintlify.yml", } { workflow := readCIWorkflow(t, file) diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index 47a5b8269..477d68f44 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -54,9 +54,8 @@ func TestPullDoltImageRetriesTransientFailures(t *testing.T) { func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { wantCalls := map[string]int{ - "pr.yml": 2, // test-domain-uow, contract-corpus - "pr-risk.yml": 3, - "regression.yml": 1, + "pr.yml": 2, // test-domain-uow, contract-corpus + "pr-risk.yml": 3, } workflowsDir := filepath.Join(sourceRepoRoot(t), ".github", "workflows") diff --git a/tests/regression/BUILD.bazel b/tests/regression/BUILD.bazel new file mode 100644 index 000000000..ee7410d1a --- /dev/null +++ b/tests/regression/BUILD.bazel @@ -0,0 +1,71 @@ +load("@rules_go//go:def.bzl", "go_test") +load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") + +# The differential regression suite (formerly regression.yml, which ran +# ./tests/regression with the regression build tag): every scenario runs against the +# catalog-pinned baseline release and against the candidate, and their +# normalized state is diffed. The baseline label must name the release in +# BASELINE_VERSION (TestMain checks the binary's own version against it). The +# candidate is the non-race bd_for_tests, built like the job's +# `go build -tags gms_pure_go ./cmd/bd`. Both bd's talk to a hermetic dolt +# sql-server TestMain starts from the pinned dolt (BEADS_TEST_DOLT_SERVER=local) +# where the job pulled the dolt-sql-server image; a server that cannot start +# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1). The Go binary shards +# itself over the top-level tests, so each runs in exactly one shard. +# +# Tags: +# dolt-server-cmd: starts a hermetic dolt sql-server and needs the +# integration build tag; selected by --config=doltserver-cmd (bazel.yml's +# bazel-cmd-dolt lane) and excluded from every other lane, whose +# configurations either drop both test files or set BEADS_TEST_SKIP=dolt. +go_test( + name = "regression_test", + size = "large", + # --config=doltserver-cmd's -test.timeout=19m fires (with a goroutine + # dump) before Bazel's 1200s action limit. + timeout = "eternal", + srcs = [ + "regression_test.go", + "scenarios_test.go", + ], + data = [ + "BASELINE_VERSION", + "//cmd/bd:bd_for_tests", + "@bd_releases//:v0.49.6", + ], + env = { + "BD_REGRESSION_BASELINE_BIN": "$(rlocationpath @bd_releases//:v0.49.6)", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, + shard_count = 8, + tags = ["dolt-server-cmd"], + target_compatible_with = LINUX_AMD64, + deps = [ + "//internal/testutil", + "//internal/testutil/bazeltest", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tests/regression/regression_tag_test.go b/tests/regression/regression_tag_test.go new file mode 100644 index 000000000..9e37a6454 --- /dev/null +++ b/tests/regression/regression_tag_test.go @@ -0,0 +1,8 @@ +//go:build regression + +package regression + +// Only -tags=regression compiles this file: it marks the build as an explicit +// request for the suite (see testMainInner). Gazelle never adds it to the +// Bazel target, which runs the suite because it runs under Bazel. +func init() { builtWithRegressionTag = true } diff --git a/tests/regression/regression_test.go b/tests/regression/regression_test.go index aef46f04a..08500bf67 100644 --- a/tests/regression/regression_test.go +++ b/tests/regression/regression_test.go @@ -1,4 +1,4 @@ -//go:build regression +//go:build regression || integration // Package regression implements differential testing between a pinned baseline // bd binary (v0.49.6) and the current worktree build. Each test scenario runs @@ -8,6 +8,15 @@ // // Run: go test -tags=regression -timeout=10m ./tests/regression/... // Or: make test-regression +// CI: bazel test //tests/regression:regression_test --config=doltserver-cmd +// +// Under Bazel the files build under the integration tag (gazelle generates +// the target from them) and both binaries are declared inputs: the candidate +// is //cmd/bd:bd_for_tests (BEADS_TEST_BD_BINARY) and the baseline is the +// catalog-pinned release @bd_releases//: +// (BD_REGRESSION_BASELINE_BIN), so the suite needs no Go toolchain and no +// network. A plain `go test -tags=integration ./...` compiles the files too +// but skips the suite: only -tags=regression (or Bazel) runs it. package regression import ( @@ -31,8 +40,13 @@ import ( "time" "github.com/steveyegge/beads/internal/testutil" + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) +// builtWithRegressionTag is set by regression_tag_test.go, which only +// -tags=regression compiles. +var builtWithRegressionTag bool + // baselineBin is the path to the pinned baseline bd binary. var baselineBin string @@ -54,6 +68,10 @@ func testMainInner(m *testing.M) int { fmt.Fprintln(os.Stderr, "regression tests not yet supported on Windows (zip extraction needed)") return 0 } + if !builtWithRegressionTag && !bazeltest.IsBazel() { + fmt.Fprintln(os.Stderr, "SKIP: differential regression suite runs only with -tags=regression or under Bazel") + return 0 + } // Start an isolated Dolt server so regression tests don't pollute // the production database on port 3307. @@ -84,14 +102,23 @@ func testMainInner(m *testing.M) int { return 1 } - // Build candidate from current worktree - candidateBin = filepath.Join(tmpDir, "bd-candidate") - fmt.Fprintln(os.Stderr, "Building candidate binary...") - if err := buildCandidate(candidateBin); err != nil { - fmt.Fprintf(os.Stderr, "building candidate: %v\n", err) + // Candidate: the prebuilt bd (required under Bazel), else a build of the + // current worktree. + candidateBin, err = bazeltest.PrebuiltBD() + if err != nil { + fmt.Fprintf(os.Stderr, "candidate binary: %v\n", err) os.RemoveAll(tmpDir) return 1 } + if candidateBin == "" { + candidateBin = filepath.Join(tmpDir, "bd-candidate") + fmt.Fprintln(os.Stderr, "Building candidate binary...") + if err := buildCandidate(candidateBin); err != nil { + fmt.Fprintf(os.Stderr, "building candidate: %v\n", err) + os.RemoveAll(tmpDir) + return 1 + } + } // Get baseline (env override > cache > download) fmt.Fprintln(os.Stderr, "Getting baseline binary...") @@ -163,20 +190,25 @@ func icuPrefixPath() string { return strings.TrimSpace(string(out)) } +// baselineBinEnv overrides the baseline binary. Under Bazel it is required +// and holds the rlocationpath of the pinned release. +const baselineBinEnv = "BD_REGRESSION_BASELINE_BIN" + func getBaseline() (string, error) { - if bin := os.Getenv("BD_REGRESSION_BASELINE_BIN"); bin != "" { + if bazeltest.IsBazel() { + return pinnedBaseline() + } + if bin := os.Getenv(baselineBinEnv); bin != "" { if _, err := os.Stat(bin); err != nil { - return "", fmt.Errorf("BD_REGRESSION_BASELINE_BIN=%q: %w", bin, err) + return "", fmt.Errorf("%s=%q: %w", baselineBinEnv, bin, err) } return bin, nil } - versionFile := filepath.Join(findModuleRoot(), "tests", "regression", "BASELINE_VERSION") - data, err := os.ReadFile(versionFile) + version, err := baselineVersion(findModuleRoot()) if err != nil { - return "", fmt.Errorf("reading BASELINE_VERSION: %w", err) + return "", err } - version := strings.TrimSpace(string(data)) cacheDir, err := os.UserCacheDir() if err != nil { @@ -205,6 +237,41 @@ func getBaseline() (string, error) { return cachedBin, nil } +// baselineVersion reads tests/regression/BASELINE_VERSION under root. +func baselineVersion(root string) (string, error) { + data, err := os.ReadFile(filepath.Join(root, "tests", "regression", "BASELINE_VERSION")) + if err != nil { + return "", fmt.Errorf("reading BASELINE_VERSION: %w", err) + } + return strings.TrimSpace(string(data)), nil +} + +// pinnedBaseline resolves the Bazel-declared baseline release and checks that +// it is the release BASELINE_VERSION names, so the BUILD label and the file +// cannot drift apart. +func pinnedBaseline() (string, error) { + bin, err := bazeltest.RunfileEnv(baselineBinEnv) + if err != nil { + return "", err + } + root := bazeltest.OverrideRoot() + if root == "" { + return "", fmt.Errorf("no repository root in the runfiles for BASELINE_VERSION") + } + version, err := baselineVersion(root) + if err != nil { + return "", err + } + out, err := exec.Command(bin, "version").CombinedOutput() + if err != nil { + return "", fmt.Errorf("%s version: %w\n%s", bin, err, out) + } + if bare := strings.TrimPrefix(version, "v"); !strings.Contains(" "+string(out)+" ", " "+bare+" ") { + return "", fmt.Errorf("baseline %s reports %q, want BASELINE_VERSION %s", bin, strings.TrimSpace(string(out)), version) + } + return bin, nil +} + func downloadAndExtract(url, destPath string) error { resp, err := http.Get(url) //nolint:gosec // URL constructed from trusted constants if err != nil { diff --git a/tests/regression/scenarios_test.go b/tests/regression/scenarios_test.go index 278a82276..bed18b84b 100644 --- a/tests/regression/scenarios_test.go +++ b/tests/regression/scenarios_test.go @@ -1,4 +1,4 @@ -//go:build regression +//go:build regression || integration package regression diff --git a/tools/bazel/bd_releases.bzl b/tools/bazel/bd_releases.bzl new file mode 100644 index 000000000..2e52fc58a --- /dev/null +++ b/tools/bazel/bd_releases.bzl @@ -0,0 +1,141 @@ +"""Historical bd release binaries for Bazel tests, pinned by the release catalog. + +scripts/migration-test/release-catalog.json is the reviewed universe of +historical releases. Every version in it whose GitHub release records a +linux_amd64 asset (github_release.linux_amd64_asset: name and sha256 digest) +becomes one repository, bd_release_, holding that release's `bd`. +Nothing is downloaded until a test declares the binary as data, and the +download is verified against the catalog's digest, so tests that run historical +releases need no network and no checksum code of their own. + +Tests use the hub repository, @bd_releases: + + data = ["@bd_releases//:v0.49.6"], + env = {"BD": "$(rlocationpath @bd_releases//:v0.49.6)"}, + +and @bd_releases//:versions.bzl lists every pinned version (VERSIONS), so a +BUILD file can generate one target per release. + +Adding a release is a catalog change (regenerate it with +scripts/migration-test/catalog); this file and MODULE.bazel stay as they are. + +The catalog pins linux/amd64 assets only: a test that runs one sets +target_compatible_with = LINUX_AMD64 (from this file), so `bazel build //...` +skips it on other hosts. +""" + +_RELEASE_URL = "https://github.com/gastownhall/beads/releases/download/{version}/{asset}" + +_RELEASE_BUILD = """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["bd"]) +""" + +# target_compatible_with for a test that runs a pinned release. +LINUX_AMD64 = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", +] + +def _bd_release_impl(rctx): + rctx.download_and_extract( + url = rctx.attr.url, + sha256 = rctx.attr.sha256, + # Ties the repository-cache entry to this URL (and so this version). + canonical_id = rctx.attr.url, + ) + if not rctx.path("bd").exists: + fail("release asset {} holds no top-level bd binary".format(rctx.attr.url)) + rctx.file("BUILD.bazel", _RELEASE_BUILD) + +bd_release = repository_rule( + implementation = _bd_release_impl, + attrs = { + "sha256": attr.string(mandatory = True, doc = "sha256 of the release archive."), + "url": attr.string(mandatory = True, doc = "Release archive URL."), + }, + doc = "Downloads one historical bd release archive and exports its `bd` binary.", +) + +def _repo_name(version): + return "bd_release_" + version.replace(".", "_").replace("-", "_") + +def _hub_impl(rctx): + lines = [ + "# Generated by tools/bazel/bd_releases.bzl from the release catalog.", + "package(default_visibility = [\"//visibility:public\"])", + "", + "exports_files([\"versions.bzl\"])", + "", + ] + for version in rctx.attr.versions: + lines.append("""alias( + name = "{version}", + actual = "@{repo}//:bd", +) +""".format(version = version, repo = _repo_name(version))) + rctx.file("BUILD.bazel", "\n".join(lines)) + rctx.file("versions.bzl", "\n".join([ + "\"\"\"Every historical bd release @bd_releases pins (from the release catalog), oldest first.\"\"\"", + "", + "VERSIONS = [", + ] + [" \"{}\",".format(v) for v in rctx.attr.versions] + ["]", ""])) + +_bd_releases_hub = repository_rule( + implementation = _hub_impl, + attrs = { + "versions": attr.string_list(mandatory = True), + }, + doc = "Hub repository: one alias per pinned release, plus versions.bzl.", +) + +def catalog_releases(catalog): + """Returns [(version, asset name, sha256)] for every catalog release with a linux_amd64 asset. + + Args: + catalog: the decoded release-catalog.json. + + Returns: + A list of (version, asset, sha256) tuples in catalog order. + """ + releases = [] + for entry in catalog["versions"]: + asset = entry.get("github_release", {}).get("linux_amd64_asset") + if asset == None: + continue + digest = asset["digest"] + if not digest.startswith("sha256:"): + fail("release {}: asset digest {} is not sha256".format(entry["version"], digest)) + releases.append((entry["version"], asset["name"], digest[len("sha256:"):])) + return releases + +def _bd_releases_impl(mctx): + catalogs = [tag.path for mod in mctx.modules if mod.is_root for tag in mod.tags.catalog] + if len(catalogs) != 1: + fail("bd_releases needs exactly one catalog tag in the root module, got {}".format(len(catalogs))) + catalog = json.decode(mctx.read(catalogs[0])) + versions = [] + for version, asset, sha256 in catalog_releases(catalog): + bd_release( + name = _repo_name(version), + url = _RELEASE_URL.format(version = version, asset = asset), + sha256 = sha256, + ) + versions.append(version) + _bd_releases_hub(name = "bd_releases", versions = versions) + return mctx.extension_metadata( + root_module_direct_deps = ["bd_releases"], + root_module_direct_dev_deps = [], + reproducible = True, + ) + +bd_releases = module_extension( + implementation = _bd_releases_impl, + tag_classes = { + "catalog": tag_class(attrs = { + "path": attr.label(mandatory = True, doc = "The release catalog JSON."), + }), + }, + doc = "Declares one repository per historical bd release in the release catalog, and the @bd_releases hub.", +)