beads/scripts/ci_workflow_test.go
Julian Knutsen 17127ee544
fix(ci-policy): close review leftovers from #7485/#7486 (#7491)
* fix(ci-policy): close review leftovers from #7485/#7486

Follow-ups from maintainer review on the host-tool exec/shell policy
tests and the bazel.yml integration gate:

- host_tool_shell_policy_test.go: recognize indented function headers
  and `function name {` forms when finding a function's closing brace,
  so scripts/test.sh's indented cleanup_shared_server() no longer
  mis-attributes its trailing lines to the wrong function; classify
  same-line `command -v gofmt && gofmt` (and `go`) as the guarded
  presence-check idiom instead of flagging it.
- host_tool_exec_policy_test.go: make condPolarity reject mixed-polarity
  && / || compounds (one recognized side, one not) instead of guessing
  at the recognized side's polarity, which was unsound in both
  directions; add pinning tests documenting the exec-before-gate and
  function-value-helper cases as known, safety-conservative limitations
  rather than bugs. Fixing this exposed a real ungated fallback in
  cmd/bd/scripttest_test.go (a `go build` that could still run under
  Bazel on a prebuilt-binary basename mismatch); fixed by symlinking the
  prebuilt binary under the expected name instead of rebuilding it.
- bazel.yml: bazel-server-storage now honors the `integration` input
  like bazel-integration/bazel-cmd-dolt do, so an explicit "off" skips
  it too (latent today: no caller passes "off"). Updated
  ci_workflow_test.go's policy tests to match.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* fix(ci-policy): close should-fix B (go command -v guard evasion) on #7491

shellCommandVGuard previously excused go the same as gofmt, but under an
exit-127 host-tool stub `command -v` finds the stub and the guarded exec
still runs, and on a worker genuinely lacking go the exec was silently
skipped -- the worker-dependent behavior this policy exists to ban. Drop
go from the guard entirely, require the guarded exec to sit directly after
the guard's own `&&` (no intervening `;`, `||`, `|`, or leading `!`), and
require an exact gofmt match so a same-prefixed sibling like `gofmt-foo`/
`go-junit-report` can't borrow the guard. Add every should-fix B shape as
a fixture that must still be flagged, plus the generalized gofmt-side
cases (leading `!`, exact-name, post-`;` exec).

Also: correct host_tool_exec_policy_test.go's doc claim that a
function-value helper call site is "not a soundness hole" -- mixed with
one gated direct call site, the invisible function-value call site's
ungated-ness is silently lost and the combined verdict flips to gated;
pinned as review M22. And recognize a function closed by `} >&2` as
ended, so abort-style safety computed inside it does not leak into the
rest of the file (no scoped script has this shape today, but a bare `}`
equality check would silently miss it the moment one does).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-11 01:22:48 +00:00

6043 lines
277 KiB
Go

package scripts_test
import (
"encoding/base64"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"reflect"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
func TestCIWorkflowArtifactOwnership(t *testing.T) {
for _, workflowName := range []string{"pr.yml", "main.yml"} {
t.Run(workflowName, func(t *testing.T) {
workflow := readCIWorkflow(t, workflowName)
// The PR Policy and lint wrappers are retired: their checks are
// Bazel tests (TestPRPolicyChecksRunAsBazelTargets) and nogo
// (TestLintAndVetRunAsNogo), so no job may run them.
for name, job := range workflow.Jobs {
for _, step := range job.Steps {
for _, forbidden := range []string{"golangci-lint", "make ci-pr-policy", "make ci-pr-lint"} {
if strings.Contains(step.Run, forbidden) {
t.Errorf("%s job %s step %q runs %q; its checks run as Bazel tests", workflowName, name, step.Name, forbidden)
}
}
}
}
for _, retired := range []string{"pr-policy-wrapper", "pr-lint-wrapper", "go-vet-cache", "scripts-go-checks", "lint"} {
if _, found := workflow.Jobs[retired]; found {
t.Errorf("%s has a %q job; its checks run in bazel.yml's test lane", workflowName, retired)
}
}
})
}
}
// TestPullRequestWorkflowsTriggerOnHotfixBranches is the #7148 regression.
// pr.yml's own comment states why release/** is in the pull_request
// trigger: "release-prep and release-fix PRs are based on the release
// branch, not on main. Without it they run almost no CI and the first
// full signal is the tag build, where a failure burns the tag." A hotfix
// backport PR (hotfix/1.3.1 and its siblings) is in exactly that position
// — based on a branch other than main — so it needs the same trigger,
// alongside release/**, not instead of it.
func TestPullRequestWorkflowsTriggerOnHotfixBranches(t *testing.T) {
type triggers struct {
On struct {
PullRequest struct {
Branches []string `yaml:"branches"`
} `yaml:"pull_request"`
} `yaml:"on"`
}
root := sourceRepoRoot(t)
for _, name := range []string{
"pr.yml", prRiskWorkflowName,
} {
t.Run(name, func(t *testing.T) {
var doc triggers
if err := yaml.Unmarshal([]byte(readPolicyFile(t, root, ".github/workflows/"+name)), &doc); err != nil {
t.Fatal(err)
}
branches := doc.On.PullRequest.Branches
if !slices.Contains(branches, "release/**") {
t.Fatalf("%s pull_request.branches = %v, want release/** present (this test asserts the baseline it extends)", name, branches)
}
if !slices.Contains(branches, "hotfix/**") {
t.Errorf("%s pull_request.branches = %v, want hotfix/** alongside release/** — a hotfix backport PR is based on a branch other than main, same as a release-prep PR", name, branches)
}
})
}
}
func TestPRCIGateDropsRetiredPolicyAndLintJobs(t *testing.T) {
gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate")
gateEnv := gate.step(t, "Evaluate CI gate").Env
// The retired lint and vet jobs (now nogo in the Bazel test lane, which
// gates through BAZEL_TEST) must not linger in the gate.
for job, token := range map[string]string{"pr-lint-wrapper": "PR_LINT_WRAPPER", "scripts-go-checks": "SCRIPTS_GO_CHECKS"} {
if contains(gate.Needs, job) {
t.Errorf("ci-gate needs the retired %s: %v", job, gate.Needs)
}
if _, found := gateEnv[token]; found {
t.Errorf("ci-gate env still sets %s", token)
}
if contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), token) {
t.Errorf("ci-gate CI_GATE_REQUIRED still lists %s", token)
}
}
// The retired PR Policy job's checks gate through BAZEL_TEST (the Bazel
// `test` lane), so its token must not linger.
if contains(gate.Needs, "pr-policy-wrapper") {
t.Errorf("ci-gate needs the retired pr-policy-wrapper: %v", gate.Needs)
}
if _, found := gateEnv["PR_POLICY_WRAPPER"]; found {
t.Errorf("ci-gate env still sets PR_POLICY_WRAPPER")
}
if contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "PR_POLICY_WRAPPER") {
t.Errorf("ci-gate CI_GATE_REQUIRED still lists PR_POLICY_WRAPPER")
}
if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "BAZEL_TEST") {
t.Errorf("ci-gate CI_GATE_REQUIRED does not include BAZEL_TEST, the lane that now runs the policy checks")
}
// F5.1/F5.4: the deleted `lint` job's LINT id must not linger in the gate.
if contains(gate.Needs, "lint") {
t.Errorf("ci-gate needs %q, which no longer exists: %v", "lint", gate.Needs)
}
if contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "LINT") {
t.Error("ci-gate CI_GATE_REQUIRED still includes LINT")
}
if _, found := gateEnv["LINT"]; found {
t.Error("ci-gate env still sets LINT")
}
}
// TestReleaseCrossCompileRunsInItsOwnBazelLane pins the release-target
// cross-compilation gate into bazel.yml's bazel-release-cross lane, whose
// job.status ci-gate requires (BAZEL_RELEASE_CROSS), and keeps it from
// running twice: not back in bazel-pure (where it was a step on the longest
// cached lane), and not as the retired pr.yml `go build` job beside it.
func TestReleaseCrossCompileRunsInItsOwnBazelLane(t *testing.T) {
const (
retiredJob = "check-release-target-cross-compilation"
retiredToken = "CHECK_RELEASE_TARGET_CROSS_COMPILATION"
stepName = "Cross-compile every release target (--config=release-cross)"
script = "./scripts/ci/bazel-release-cross-compile.sh"
)
pr := readCIWorkflow(t, "pr.yml")
if _, ok := pr.Jobs[retiredJob]; ok {
t.Errorf("pr.yml still defines %s; bazel.yml's %s lane runs the release cross-compilation", retiredJob, bazelReleaseCrossJobName)
}
gate := pr.job(t, "ci-gate")
gateEnv := gate.step(t, "Evaluate CI gate").Env
if contains(gate.Needs, retiredJob) || gateEnv[retiredToken] != "" || contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), retiredToken) {
t.Errorf("ci-gate still wires the retired %s job", retiredJob)
}
if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "BAZEL_RELEASE_CROSS") {
t.Errorf("ci-gate CI_GATE_REQUIRED does not include BAZEL_RELEASE_CROSS, the lane that cross-compiles the release targets")
}
bazel := readCIWorkflow(t, bazelWorkflowName)
for name, job := range bazel.Jobs {
if name == bazelReleaseCrossJobName {
continue
}
for _, step := range job.Steps {
if strings.Contains(step.Run, script) {
t.Errorf("%s step %q runs %s too; only %s cross-compiles the release targets", name, step.Name, script, bazelReleaseCrossJobName)
}
}
}
cross := bazel.job(t, bazelReleaseCrossJobName)
pure := bazel.job(t, bazelPureJobName)
// Not pure's runs-on: pure reads the remote repo contents cache in mode
// cache (ubuntu-24.04, bazelRRCCacheReaderRunsOn), the release lane does
// not (its target platforms are not seeded).
if cross.RunsOn != bazelLaneRunsOn || cross.If != pure.If || !slices.Equal(cross.Needs, pure.Needs) {
t.Errorf("%s runs-on/if/needs = %q / %q / %v, want %q / %s's %q / %v",
bazelReleaseCrossJobName, cross.RunsOn, cross.If, cross.Needs, bazelLaneRunsOn, bazelPureJobName, pure.If, pure.Needs)
}
run := cross.step(t, stepName).Run
for _, required := range []string{
"set -euo pipefail",
script,
} {
if !strings.Contains(run, required) {
t.Errorf("%s step %q does not contain %q:\n%s", bazelReleaseCrossJobName, stepName, required, run)
}
}
// The transition is what makes every platform's build pure Go (cgo off,
// as CGO_ENABLED=0 go build) for the platform the script names.
bzl := readPolicyFile(t, bazelPolicyRoot(t), "tools/bazel/release_cross.bzl")
for _, required := range []string{
`"//command_line_option:platforms": ["@rules_go//go/toolchain:" + platform],`,
`"@rules_go//go/config:pure": True,`,
`inputs = ["//tools/bazel:release_platforms"],`,
} {
if !strings.Contains(bzl, required) {
t.Errorf("tools/bazel/release_cross.bzl does not contain %q", required)
}
}
}
// TestReleaseTargetCrossCompilationMatrixMatchesGoreleaser keeps
// scripts/ci/release-targets.txt (the single source of truth for the
// platforms scripts/ci/bazel-release-cross-compile.sh builds in bazel.yml's
// pure-Go lane) and the set of shipped release targets in lockstep. Without
// a guard a newly added release target -- the way freebsd/amd64 once was --
// is silently uncovered while a green cross-compilation check still stands.
// That is worse than having no check at all, because the check's existence
// implies the coverage it has quietly lost.
func TestReleaseTargetCrossCompilationMatrixMatchesGoreleaser(t *testing.T) {
const manifestPath = "scripts/ci/release-targets.txt"
// darwin/amd64 and darwin/arm64 are shipped release targets that are
// deliberately absent from .goreleaser.yml's builds: release.yml's
// goreleaser-macos job builds them natively at CGO_ENABLED=1 for embedded
// Dolt support, as .goreleaser.yml's own comment records.
want := map[string]string{
"darwin/amd64": "release.yml goreleaser-macos",
"darwin/arm64": "release.yml goreleaser-macos",
}
for _, build := range readGoreleaserBuilds(t) {
for _, goos := range build.GOOS {
for _, goarch := range build.GOARCH {
want[goos+"/"+goarch] = ".goreleaser.yml " + build.ID
}
}
}
raw, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), manifestPath))
if err != nil {
t.Fatalf("read %s: %v", manifestPath, err)
}
got := make(map[string]bool)
for _, line := range strings.Split(string(raw), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) != 2 {
t.Fatalf("%s: malformed row %q, want 'GOOS GOARCH'", manifestPath, line)
}
got[fields[0]+"/"+fields[1]] = true
}
for target, source := range want {
if !got[target] {
t.Errorf("release target %s (%s) is not covered by %s", target, source, manifestPath)
}
}
for target := range got {
if _, ok := want[target]; !ok {
t.Errorf("%s lists %s, which is not a shipped release target", manifestPath, target)
}
}
}
type goreleaserBuild struct {
ID string `yaml:"id"`
GOOS []string `yaml:"goos"`
GOARCH []string `yaml:"goarch"`
Env []string `yaml:"env"`
}
func readGoreleaserBuilds(t *testing.T) []goreleaserBuild {
t.Helper()
path := filepath.Join(sourceRepoRoot(t), ".goreleaser.yml")
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var config struct {
Builds []goreleaserBuild `yaml:"builds"`
}
if err := yaml.Unmarshal(data, &config); err != nil {
t.Fatalf("parse %s: %v", path, err)
}
if len(config.Builds) == 0 {
t.Fatalf("%s declares no builds", path)
}
return config.Builds
}
// Exclude read-permission coverage is required, not best effort: Bazel's
// PR-core lane (bazel.yml's `test`, the only PR run of PR Core's tests)
// requires it, and pr.yml's gate requires that lane.
func TestPRCoreRequiresExcludeReadPermissionCoverage(t *testing.T) {
if !contains(bazelPRLaneRCLines["prcore"], "test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1") {
t.Error("Bazel's PR-core lane must require actual exclude read-permission coverage")
}
gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate")
evaluate := gate.step(t, "Evaluate CI gate")
if gate.If != "${{ always() }}" || gate.ContinueOnError || evaluate.If != "" || (evaluate.ContinueOnError != nil && evaluate.ContinueOnError != false) {
t.Error("CI gate must propagate required Bazel PR-core failures")
}
if !contains(gate.Needs, "bazel") || !contains(strings.Fields(evaluate.Env["CI_GATE_REQUIRED"]), "BAZEL_TEST") {
t.Error("CI gate must require the Bazel PR-core lane (BAZEL_TEST)")
}
}
func TestPRComplexityReportIsAdvisoryAndBestEffort(t *testing.T) {
workflow := readCIWorkflow(t, "pr.yml")
job := workflow.job(t, "advisory-reports")
// F7a: advisory-reports (examples build + complexity) moved to a
// same-repo Blacksmith runner and gained a generous job-level timeout
// (TestSameRepoBlacksmithRunners requires one of every moved job) that is
// a backstop against a stuck runner, not a realistic ceiling - every
// step underneath still carries its own tight timeout/continue-on-error.
if job.RunsOn != sameRepoBlacksmith2vcpu || job.TimeoutMinutes != 60 || job.ContinueOnError {
t.Errorf("advisory-reports job must run on same-repo Blacksmith with a backstop timeout/no job continue-on-error: runs-on=%q timeout=%d continue=%v", job.RunsOn, job.TimeoutMinutes, job.ContinueOnError)
}
if contains(job.Needs, "ci-gate") {
t.Errorf("complexity report unexpectedly depends on ci-gate: %v", job.Needs)
}
for _, name := range []string{"Set up Go", "Install gocyclo", "Generate complexity report", "Annotate unavailable complexity report", "Upload complexity report"} {
step := job.step(t, name)
if step.TimeoutMinutes <= 0 || step.ContinueOnError != true {
t.Errorf("complexity step %q is not bounded/best-effort: timeout=%d continue=%v", name, step.TimeoutMinutes, step.ContinueOnError)
}
}
checkout := job.Steps[0]
if checkout.TimeoutMinutes <= 0 || checkout.ContinueOnError != true {
t.Errorf("complexity checkout is not bounded/best-effort: timeout=%d continue=%v", checkout.TimeoutMinutes, checkout.ContinueOnError)
}
report := job.step(t, "Generate complexity report")
if report.ID != "generate-complexity" || report.If != "always()" || !strings.Contains(report.Run, "complexity.sh diff") || !strings.Contains(report.Run, "COMPLEXITY_BASE_REF=origin/main") {
t.Errorf("complexity report step missing diff/always contract: id=%q if=%q run=%q", report.ID, report.If, report.Run)
}
annotate := job.step(t, "Annotate unavailable complexity report")
if annotate.If != "always()" || !strings.Contains(annotate.Run, "::warning") {
t.Errorf("complexity annotation step missing always/warning contract: if=%q run=%q", annotate.If, annotate.Run)
}
gate := workflow.job(t, "ci-gate")
if contains(gate.Needs, "advisory-reports") {
t.Errorf("ci-gate must not require advisory complexity report: %v", gate.Needs)
}
}
func TestPRWorkflowExercisesNativeUserConfigDiagnostics(t *testing.T) {
workflow := readCIWorkflow(t, "pr.yml")
job := workflow.job(t, "pr-preflight-platforms")
// The benchmark-environment test owns the shared job and matrix shape.
wantHosts := map[string]string{"macos-latest": "darwin", "windows-latest": "windows"}
gotHosts := make(map[string][]string)
for _, tuple := range job.Strategy.Matrix.Include {
gotHosts[tuple.OS] = append(gotHosts[tuple.OS], tuple.ExpectedGOOS)
}
for host, want := range wantHosts {
if got := gotHosts[host]; len(got) != 1 || got[0] != want {
t.Fatalf("native host %s = %v, want exactly one %s", host, got, want)
}
}
step := job.step(t, "Check native user config diagnostics")
// F7b review fix (S1): the Windows shard split this step's `if:` used to
// be gated on (matrix.shard == 1) was reverted (see pr-preflight-platforms'
// own comment in pr.yml) because it measurably failed to help: shard 2
// only ever held 15-25s of work while shard 1 stayed at 450-480s. The
// step is unconditional again, same as every OS's leg.
if step.If != "" || step.Shell != "bash" || step.Env["CGO_ENABLED"] != "0" ||
(step.ContinueOnError != nil && step.ContinueOnError != false) {
t.Fatalf("native diagnostic step is conditional, optional, or uses the wrong host: %+v", step)
}
assertStepRunsExactly(t, job, step.Name, "bash scripts/ci/test-user-config-diagnostic.sh '${{ matrix.expected_goos }}'")
assertStepsBefore(t, job, []string{"Set up Go", "Restore Go module cache"}, []string{step.Name})
gate := workflow.job(t, "ci-gate")
gateEnv := gate.step(t, "Evaluate CI gate").Env
if !contains(gate.Needs, "pr-preflight-platforms") ||
gateEnv["PR_PREFLIGHT_PLATFORMS"] != "${{ needs.pr-preflight-platforms.result }}" ||
!contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "PR_PREFLIGHT_PLATFORMS") {
t.Fatal("native diagnostic platform results must feed the required aggregate gate")
}
}
func TestPRWorkflowRequiresNativeInitGatewayCredential(t *testing.T) {
const (
jobName = "pr-preflight-platforms"
stepCommand = `bash scripts/ci/test-init-gateway-credential.sh "$RUNNER_OS"`
gateKey = "PR_PREFLIGHT_PLATFORMS"
)
workflow := readCIWorkflow(t, "pr.yml")
job := workflow.job(t, jobName)
if job.RunsOn != platformsMatrixRunsOn || !equalStrings(matrixIncludeHosts(job.Strategy.Matrix.Include), []string{"macos-latest", "windows-latest"}) {
t.Errorf("credential shell boundary requires the native macOS/Windows matrix: runner=%q include=%v", job.RunsOn, job.Strategy.Matrix.Include)
}
if job.If != "" || job.ContinueOnError {
t.Errorf("credential process job is bypassable: if=%q continue-on-error=%v", job.If, job.ContinueOnError)
}
matchingSteps := 0
for _, step := range job.Steps {
if strings.TrimSpace(step.Run) != stepCommand {
continue
}
matchingSteps++
if step.Shell != "bash" || step.Env["RUNNER_OS"] != "" {
t.Errorf("credential driver needs Bash and the native runner OS: shell=%q env=%v", step.Shell, step.Env)
}
// F7b review fix (S1): common to all three hosts; the shard split this
// step's `if:` used to be gated on (matrix.shard == 1) was reverted
// (see pr-preflight-platforms' own comment in pr.yml), so the step
// is unconditional again.
if step.If != "" || (step.ContinueOnError != nil && step.ContinueOnError != false) {
t.Errorf("gateway credential step is bypassable: if=%q continue-on-error=%v", step.If, step.ContinueOnError)
}
}
if matchingSteps != 1 {
t.Fatalf("native preflight job has %d gateway credential commands, want 1", matchingSteps)
}
gate := workflow.job(t, "ci-gate")
gateEnv := gate.step(t, "Evaluate CI gate").Env
if !contains(gate.Needs, jobName) || gateEnv[gateKey] != "${{ needs.pr-preflight-platforms.result }}" ||
!contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), gateKey) {
t.Errorf("ci-gate does not require the native credential process lane: needs=%v %s=%q required=%q",
gate.Needs, gateKey, gateEnv[gateKey], gateEnv["CI_GATE_REQUIRED"])
}
}
// Every run step in the three-host preflight/doc-freshness jobs invokes Go,
// and the jobs restore a non-race GOCACHE precisely so those invocations are
// incremental. A run step without GOCACHE silently falls back to the runner's
// empty default cache: "Exercise native init gateway credential shell" did,
// and cold-compiled ./cmd/bd on every PR (210-240s of a ~370s macos-latest
// leg, ~120s on Windows) right after the hook-timeout step had compiled the
// same test binary into the restored cache.
func TestPRPreflightPlatformsRunStepsUseRestoredGoBuildCache(t *testing.T) {
workflow := readCIWorkflow(t, "pr.yml")
for _, jobName := range []string{"pr-preflight-platforms", "check-doc-freshness-platforms"} {
job := workflow.job(t, jobName)
restore := job.step(t, "Restore non-race Go build cache")
if restore.With["path"] != goBuildCachePath("non-race") {
t.Errorf("%s restores the non-race cache to %q, want %q", jobName, restore.With["path"], goBuildCachePath("non-race"))
}
runSteps := 0
for _, step := range job.Steps {
if strings.TrimSpace(step.Run) == "" {
continue
}
runSteps++
assertGoCacheEnv(t, job, step.Name, "non-race")
assertStepsBefore(t, job, []string{restore.Name}, []string{step.Name})
}
if runSteps == 0 {
t.Errorf("%s has no run steps", jobName)
}
}
}
// main.yml's test job runs the macOS -race -short suite on Blacksmith macOS
// (blacksmithMacOSLabel, the image pr.yml's same-repo macOS legs use), and
// restores the module and non-race caches main.yml's
// blacksmith-macos-go-build-cache Blacksmith leg seeds: the same path and a
// restore-keys prefix of that saver's key. It writes only its own race cache.
func TestMainMacOSTestJobRestoresBlacksmithMacOSSaverCaches(t *testing.T) {
workflow := readCIWorkflow(t, "main.yml")
job := workflow.job(t, "test")
if want := "${{ '" + blacksmithMacOSLabel + "' }}"; job.RunsOn != want {
t.Errorf("main.yml test runs-on = %q, want %q", job.RunsOn, want)
}
if job.TimeoutMinutes == 0 {
t.Error("main.yml test has no timeout-minutes")
}
saver := workflow.job(t, "blacksmith-macos-go-build-cache")
for _, pair := range [][2]string{
{"Restore Go module cache", "Save Go module cache"},
{"Restore non-race Go build cache", "Save non-race Go build cache"},
} {
restore, save := job.step(t, pair[0]), saver.step(t, pair[1])
if restore.With["path"] != save.With["path"] || restore.With["restore-keys"] == "" || !strings.HasPrefix(save.With["key"], restore.With["restore-keys"]) {
t.Errorf("main.yml test %q (%q, %q) cannot reach the macOS saver's %q (%q, %q)",
pair[0], restore.With["path"], restore.With["restore-keys"], pair[1], save.With["path"], save.With["key"])
}
}
for _, step := range job.Steps {
if actionFamily(step.Uses) == cacheSaveActionFamily && step.Name != "Save race Go build cache" {
t.Errorf("main.yml test step %q saves a cache; only its own race cache may (the macOS saver owns the rest)", step.Name)
}
}
}
func TestPRWorkflowExercisesWindowsBenchmarkEnvScrubbing(t *testing.T) {
workflow := readCIWorkflow(t, "pr.yml")
job := workflow.job(t, "pr-preflight-platforms")
if job.RunsOn != platformsMatrixRunsOn {
t.Errorf("pr-preflight-platforms runs-on = %q, want the per-leg Blacksmith label", job.RunsOn)
}
if got := matrixIncludeHosts(job.Strategy.Matrix.Include); !equalStrings(got, []string{"macos-latest", "windows-latest"}) {
t.Errorf("pr-preflight-platforms matrix os = %v, want the macOS and Windows hosts (Linux runs under Bazel)", got)
}
for _, tuple := range job.Strategy.Matrix.Include {
if tuple.Runner != platformsMatrixRunners[tuple.OS] {
t.Errorf("pr-preflight-platforms %s leg runner = %q, want %q", tuple.OS, tuple.Runner, platformsMatrixRunners[tuple.OS])
}
}
if job.If != "" {
t.Errorf("pr-preflight-platforms job is conditional: %q", job.If)
}
if job.ContinueOnError {
t.Error("pr-preflight-platforms job may not continue on error")
}
step := job.step(t, "Check benchmark environment scrubbing")
// F7b: this step is Windows-only.
if step.If != "matrix.os == 'windows-latest'" {
t.Errorf("benchmark environment scrubbing selector = %q, want native Windows only", step.If)
}
if step.ContinueOnError != nil && step.ContinueOnError != false {
t.Error("benchmark environment scrubbing step may not continue on error")
}
const command = "go test -tags gms_pure_go -count=1 -run '^(TestCleanEnvUsesHostKeySemantics|TestBenchmarkCommandBuildersStripDoltEnvOverrides)$' ./scripts/repro-dolt-prod-timeouts"
if got := strings.TrimSpace(step.Run); got != command {
t.Errorf("benchmark environment scrubbing command = %q, want %q", got, command)
}
gate := workflow.job(t, "ci-gate")
gateEnv := gate.step(t, "Evaluate CI gate").Env
if gate.If != "${{ always() }}" {
t.Errorf("ci-gate condition = %q, want always() aggregation", gate.If)
}
if gate.ContinueOnError {
t.Error("ci-gate may not continue on error")
}
if !contains(gate.Needs, "pr-preflight-platforms") {
t.Errorf("ci-gate does not require pr-preflight-platforms: %v", gate.Needs)
}
if got := gateEnv["PR_PREFLIGHT_PLATFORMS"]; got != "${{ needs.pr-preflight-platforms.result }}" {
t.Errorf("ci-gate pr-preflight-platforms result = %q", got)
}
if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "PR_PREFLIGHT_PLATFORMS") {
t.Error("ci-gate required set omits pr-preflight-platforms")
}
}
func TestPRWorkflowExercisesWindowsEnvironmentHelpers(t *testing.T) {
workflow := readCIWorkflow(t, "pr.yml")
// The benchmark test owns this shared job's matrix and CI Gate propagation.
job := workflow.job(t, "pr-preflight-platforms")
step := job.step(t, "Check shared environment key semantics")
// F7b: Windows-only, alongside the other windows-only steps.
if step.If != "matrix.os == 'windows-latest'" || step.Shell != "bash" {
t.Errorf("environment helpers need native Windows Bash: if=%q shell=%q", step.If, step.Shell)
}
if step.ContinueOnError != nil && step.ContinueOnError != false {
t.Error("environment helper step may not continue on error")
}
if got := strings.TrimSpace(step.Run); got != "bash scripts/ci/test-windows-env-helpers.sh" {
t.Errorf("environment helper entrypoint = %q", got)
}
}
func TestPRPreflightPlatformsExercisesCredentialCommandFixturesOnWindows(t *testing.T) {
workflow := readCIWorkflow(t, "pr.yml")
job := workflow.job(t, "pr-preflight-platforms")
step := job.step(t, "Exercise credential command fixtures on Windows")
// F7b: Windows-only, alongside the other windows-only steps.
if step.If != "matrix.os == 'windows-latest'" {
t.Errorf("credential command fixture selector = %q, want native Windows only", step.If)
}
if step.Shell != "pwsh" {
t.Errorf("credential command fixture shell = %q, want PowerShell", step.Shell)
}
if step.ContinueOnError != nil && step.ContinueOnError != false {
t.Error("credential command fixture step may not continue on error")
}
if got := step.Env["CGO_ENABLED"]; got != "0" {
t.Errorf("credential command fixture CGO_ENABLED = %q, want 0", got)
}
// Pin the step's assertions, not just its wiring. Pinning only the packages
// and one test name left a required gate whose sole surviving guarantee was
// "go test exited 0" — which a fully skipped suite also satisfies. Dropping
// six of the nine names from $expected, deleting the fail/skip throw, or
// deleting the exactly-one-PASS loop all kept this test green. Every name
// below is a test the Windows job must actually observe passing, so a
// selector that narrows silently now fails here; widen the two together.
for _, required := range []string{
"internal/testutil/credentialcmd", "TestProtocol",
"internal/creds",
"TestCommandSourceRealShell", "TestCredentialCommandFixtureProtocol",
"internal/storage/dolt",
"TestApplyGatewayCredentialCommand", "TestApplyGatewayCredentialJSONEnvelope",
"TestApplyGatewayCredentialFailsClosed", "TestApplyGatewayCredentialPresetWins",
"TestApplyGatewayCredentialRejectsBadCharToken", "TestApplyResolvedConfigGatewayCredential",
// The two load-bearing guards inside the step body.
"'fail', 'skip'", "Expected one PASS",
} {
if !strings.Contains(step.Run, required) {
t.Errorf("credential command fixture step omits %q", required)
}
}
gate := workflow.job(t, "ci-gate")
gateEnv := gate.step(t, "Evaluate CI gate").Env
if !contains(gate.Needs, "pr-preflight-platforms") ||
gateEnv["PR_PREFLIGHT_PLATFORMS"] != "${{ needs.pr-preflight-platforms.result }}" ||
!contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "PR_PREFLIGHT_PLATFORMS") {
t.Errorf("credential command fixture job is not required by ci-gate: needs=%v result=%q required=%q",
gate.Needs, gateEnv["PR_PREFLIGHT_PLATFORMS"], gateEnv["CI_GATE_REQUIRED"])
}
}
func TestPRCIGateRequiresWindowsGlobalPrimeOverride(t *testing.T) {
workflow := readCIWorkflow(t, "pr.yml")
job := workflow.job(t, "test-windows-liveness")
step := job.step(t, "Run native Windows global Prime override")
if job.If != "" || job.ContinueOnError || step.If != "" ||
(step.ContinueOnError != nil && step.ContinueOnError != false) {
t.Fatal("native Windows global Prime override must be unconditional and required")
}
if step.Shell != "bash" || step.Env["CGO_ENABLED"] != "1" {
t.Fatal("native Windows global Prime override requires Bash and CGO")
}
if !strings.Contains(step.Run, "./scripts/test.sh") ||
!strings.Contains(step.Run, "^TestPrimeBinaryPortfolio$/^TestPrime_HookJSON_GlobalPrimeOverride$") {
t.Fatal("native Windows gate must execute the global Prime override fixture")
}
gate := workflow.job(t, "ci-gate")
evaluate := gate.step(t, "Evaluate CI gate")
env := evaluate.Env
// The native pair is the required run of these tests (ga-96smfk.22
// retired the Linux-cross-compiled "-prebuilt" twins and the
// WINDOWS_PREBUILT_REQUIRED flag): statically in CI_GATE_REQUIRED.
required := strings.Fields(env["CI_GATE_REQUIRED"])
for job, id := range map[string]string{"test-windows-liveness": "TEST_WINDOWS_LIVENESS", "worktree-remove-windows": "WORKTREE_REMOVE_WINDOWS"} {
if !contains(gate.Needs, job) || env[id] != "${{ needs."+job+".result }}" || !contains(required, id) {
t.Errorf("CI gate must require %s (%s)", job, id)
}
}
for _, retired := range []string{"windows-test-binaries", "test-windows-liveness-prebuilt", "worktree-remove-windows-prebuilt"} {
if _, ok := workflow.Jobs[retired]; ok || contains(gate.Needs, retired) {
t.Errorf("pr.yml has retired job %s", retired)
}
}
if _, ok := workflow.Env["WINDOWS_PREBUILT_REQUIRED"]; ok || strings.Contains(evaluate.Run, "PREBUILT") {
t.Error("pr.yml still carries the retired WINDOWS_PREBUILT_REQUIRED switch")
}
}
func TestPRCIGateRequiresGeneratedHookTimeoutProcessBoundary(t *testing.T) {
const (
jobName = "pr-preflight-platforms"
stepName = "Exercise generated Git hook timeout process boundary and go test's bd build fallback"
stepCommand = "go test '-tags=gms_pure_go' -count=1 -run '^(TestGeneratedHookTimeoutProcessBoundary|TestGOMODCACHENotUnderTestHome|TestGoBuildBDCommandIsCWDIndependent)$' ./cmd/bd"
gateKey = "PR_PREFLIGHT_PLATFORMS"
)
workflow := readCIWorkflow(t, "pr.yml")
job := workflow.job(t, jobName)
if job.RunsOn != platformsMatrixRunsOn || !equalStrings(matrixIncludeHosts(job.Strategy.Matrix.Include), []string{"macos-latest", "windows-latest"}) {
t.Errorf("generated-hook process job is not the required macOS/Windows matrix: runs-on=%q include=%v", job.RunsOn, job.Strategy.Matrix.Include)
}
if job.TimeoutMinutes != 20 {
t.Errorf("generated-hook process job timeout = %d minutes, want 20", job.TimeoutMinutes)
}
step := job.step(t, stepName)
// F7b review fix (S1): common to all three hosts; unconditional again
// now that the shard split is reverted (see pr-preflight-platforms' own
// comment in pr.yml).
if step.If != "" || (step.ContinueOnError != nil && step.ContinueOnError != false) || step.Shell != "bash" || step.Run != stepCommand {
t.Errorf("generated-hook process step is not required exact Bash execution: if=%q continue-on-error=%v shell=%q run=%q",
step.If, step.ContinueOnError, step.Shell, step.Run)
}
assertStepsBefore(t, job, []string{"Restore Go module cache"}, []string{stepName})
gate := workflow.job(t, "ci-gate")
gateEnv := gate.step(t, "Evaluate CI gate").Env
if !contains(gate.Needs, jobName) || gateEnv[gateKey] != "${{ needs.pr-preflight-platforms.result }}" ||
!contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), gateKey) {
t.Errorf("ci-gate does not require the three-host generated-hook lane: needs=%v %s=%q required=%q",
gate.Needs, gateKey, gateEnv[gateKey], gateEnv["CI_GATE_REQUIRED"])
}
}
func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) {
const (
workspaceBDBinary = "${{ github.workspace }}/bd"
buildCommand = "go build -v -tags gms_pure_go ./cmd/bd"
// -timeout=30m is pinned here because ./cmd/bd has outgrown `go test`'s
// 10m per-package default (#6091, and wy-5b5fbl before it — that
// default is what made this leg flaky). It sits on the invocation
// rather than in matrix.test-flags, so editing the matrix cannot
// silently drop it.
mainTestCommand = "go test -tags gms_pure_go -v -race -short -timeout=30m -skip '^TestEmbedded' ./..."
)
workflows := map[string]ciWorkflow{
"main.yml": readCIWorkflow(t, "main.yml"),
"pr.yml": readCIWorkflow(t, "pr.yml"),
}
// pr.yml's own "test-macos" job (a full-suite pre-merge race against
// main.yml's macOS leg, run on every PR to catch macOS-only regressions
// before merge) was removed under the F4 rollout: main.yml's
// push-triggered macOS job below is now the only lane that ever runs
// this suite on macOS, so no PR run exercises it pre-merge any more.
// Nothing in pr.yml should carry the workspace bd binary override
// checked in the loop below.
mainTest := workflows["main.yml"].job(t, "test")
assertStepRunsExactly(t, mainTest, "Build", buildCommand)
assertStepRunsExactly(t, mainTest, "Test", mainTestCommand)
assertStepsBefore(t, mainTest, []string{"Build"}, []string{"Test"})
for _, name := range []string{"Build", "Test"} {
if got := mainTest.step(t, name).If; got != "" {
t.Errorf("main %s condition = %q, want unconditional (the job is macOS only)", name, got)
}
}
assertStepEnvValue(t, mainTest, "Test", "BEADS_TEST_BD_BINARY", workspaceBDBinary)
for workflowName, workflow := range workflows {
for jobName, job := range workflow.Jobs {
for _, step := range job.Steps {
if step.Env["BEADS_TEST_BD_BINARY"] == workspaceBDBinary &&
!(workflowName == "main.yml" && jobName == "test" && step.Name == "Test") {
t.Errorf("%s job %q step %q has unexpected workspace bd binary override", workflowName, jobName, step.Name)
}
}
}
}
}
// TestEmbeddedDoltBatchApplySuiteRunsNonRaceUnderBazel pins the non-race
// embedded batch-apply variant that replaced nightly.yml's retired go test
// step: every race lane shrinks or skips the 1000-item apply, the 712-item
// statement-count baseline and the 458-issue batch-create equivalence
// scenarios (raceEnabled), so this target is their only full-size run. It
// must stay non-race (go_test_race_off over embeddeddolt_test; race inflates
// the embedded engine's own wall-clock), set BEADS_TEST_EMBEDDED_DOLT=1 (the
// gate skipUnlessEmbeddedDolt checks), select exactly these tests, and stay in
// the embedded tier (tag embedded), which bazel.yml runs on every PR, merge
// group, push to main and night.
func TestEmbeddedDoltBatchApplySuiteRunsNonRaceUnderBazel(t *testing.T) {
build := readPolicyFile(t, sourceRepoRoot(t), "internal/storage/embeddeddolt/BUILD.bazel")
rules := map[string]string{}
for _, r := range bazelTopRules(build) {
if m := regexp.MustCompile(`(?m)^ name = "([^"]+)",$`).FindStringSubmatch(r); m != nil {
rules[m[1]] = r
}
}
raceOff := rules["embeddeddolt_race_off"]
if !strings.HasPrefix(raceOff, "go_test_race_off(") || !strings.Contains(raceOff, `test = ":embeddeddolt_test",`) {
t.Fatalf("embeddeddolt_race_off = %q, want go_test_race_off over :embeddeddolt_test", raceOff)
}
suite := rules["embeddeddolt_batch_apply_nonrace_test"]
for _, want := range []string{
`"$(rootpath :embeddeddolt_race_off)",`,
`"-test.run=^(TestBatchApplyContract|TestLargeBatchApplyWallClock_Embedded|TestLargeBatchApplyStatementCounts712_Embedded|TestCreateBatchFastPathsMatchPerRowLarge_Embedded)$$",`,
`env = {"BEADS_TEST_EMBEDDED_DOLT": "1"},`,
`tags = ["embedded"],`,
} {
if !strings.Contains(suite, want) {
t.Errorf("embeddeddolt_batch_apply_nonrace_test lacks %s:\n%s", want, suite)
}
}
if strings.Contains(suite, ":embeddeddolt_test)") {
t.Errorf("embeddeddolt_batch_apply_nonrace_test runs the race-configured :embeddeddolt_test; want :embeddeddolt_race_off")
}
}
// TestNightlyRunsNoGoTestOnARunner: nightly.yml's only test run is the
// fresh Bazel call on rbe-west (ga-96smfk.22); its retired full-test,
// bazel-go-test-json and measure jobs ran go test on GitHub runners.
func TestNightlyRunsNoGoTestOnARunner(t *testing.T) {
nightly := readCIWorkflow(t, "nightly.yml")
for name, job := range nightly.Jobs {
for _, step := range job.Steps {
if regexp.MustCompile(`\bgo (test|build)\b|make ci-`).MatchString(step.Run) {
t.Errorf("nightly.yml job %s step %q runs %q on a runner; run it under Bazel", name, step.Name, step.Run)
}
}
if job.Uses != "" && job.Uses != "./.github/workflows/"+bazelWorkflowName {
t.Errorf("nightly.yml job %s calls %s; only %s may be called", name, job.Uses, bazelWorkflowName)
}
}
if _, err := os.Stat(filepath.Join(sourceRepoRoot(t), ".github/workflows/ci-measurements.yml")); !os.IsNotExist(err) {
t.Errorf("ci-measurements.yml is back (stat err %v); its go test measurement suites ran on GitHub runners", err)
}
}
func TestPRPreflightPlatformsRunsTestScriptPrebuiltBinaryContract(t *testing.T) {
workflow := readCIWorkflow(t, "pr.yml")
job := workflow.job(t, "pr-preflight-platforms")
if job.RunsOn != platformsMatrixRunsOn {
t.Errorf("pr-preflight-platforms runs-on = %q, want the per-leg Blacksmith label", job.RunsOn)
}
if job.If != "" || job.TimeoutMinutes != 20 {
t.Errorf("pr-preflight-platforms condition/timeout = %q/%d, want unconditional/20",
job.If, job.TimeoutMinutes)
}
if got := matrixIncludeHosts(job.Strategy.Matrix.Include); !equalStrings(got, []string{"macos-latest", "windows-latest"}) {
t.Errorf("pr-preflight-platforms matrix os = %v, want the macOS and Windows hosts", got)
}
const stepName = "Exercise test.sh prebuilt binary path"
assertStepRunsExactly(t, job, stepName,
"go test '-tags=gms_pure_go' -count=1 -run '^TestTestScriptPrebuiltBinaryContract$' ./scripts")
step := job.step(t, stepName)
// F7b review fix (S1): common to all three hosts; unconditional again
// now that the shard split is reverted (see pr-preflight-platforms' own
// comment in pr.yml).
if step.Shell != "bash" || step.If != "" || (step.ContinueOnError != nil && step.ContinueOnError != false) {
t.Errorf("%s shell/condition/continue-on-error = %q/%q/%v, want unconditional required bash",
stepName, step.Shell, step.If, step.ContinueOnError)
}
gate := workflow.job(t, "ci-gate")
gateEnv := gate.step(t, "Evaluate CI gate").Env
if !contains(gate.Needs, "pr-preflight-platforms") {
t.Errorf("ci-gate does not need pr-preflight-platforms: %v", gate.Needs)
}
if got, want := gateEnv["PR_PREFLIGHT_PLATFORMS"], "${{ needs.pr-preflight-platforms.result }}"; got != want {
t.Errorf("ci-gate PR_PREFLIGHT_PLATFORMS = %q, want %q", got, want)
}
if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "PR_PREFLIGHT_PLATFORMS") {
t.Errorf("ci-gate required set omits PR_PREFLIGHT_PLATFORMS")
}
}
func TestRepositoryTextEOLPolicyWorkflow(t *testing.T) {
workflow := readCIWorkflow(t, "pr.yml")
job := workflow.job(t, "check-doc-freshness-platforms")
if want := platformsMatrixRunsOn; job.RunsOn != want {
t.Errorf("check-doc-freshness-platforms runs-on = %q, want %q", job.RunsOn, want)
}
// The Linux leg runs in Bazel (TestDocFreshnessLinuxLegRunsInBazel);
// macOS and Windows have no remote workers and stay here.
wantMatrix := map[string]string{
"macos-latest": "darwin",
"windows-latest": "windows",
}
// Each host's include tuple names its Blacksmith label in "runner"
// (platformsMatrixRunsOn), for every PR, forks included.
wantRunnerMarker := platformsMatrixRunners
if len(job.Strategy.Matrix.OS) != 0 {
t.Errorf("check-doc-freshness-platforms retains an unbound os-list matrix: %v", job.Strategy.Matrix.OS)
}
if got, want := len(job.Strategy.Matrix.Include), len(wantMatrix); got != want {
t.Fatalf("check-doc-freshness-platforms include tuple count = %d, want %d", got, want)
}
seen := make(map[string]bool, len(wantMatrix))
for _, tuple := range job.Strategy.Matrix.Include {
wantGOOS, ok := wantMatrix[tuple.OS]
if !ok {
t.Errorf("unexpected check-doc-freshness-platforms runner tuple: %+v", tuple)
continue
}
if seen[tuple.OS] {
t.Errorf("duplicate check-doc-freshness-platforms runner tuple for %q", tuple.OS)
}
seen[tuple.OS] = true
if tuple.ExpectedGOOS != wantGOOS {
t.Errorf("runner %q expected_goos = %q, want %q", tuple.OS, tuple.ExpectedGOOS, wantGOOS)
}
if tuple.Runner != wantRunnerMarker[tuple.OS] {
t.Errorf("runner %q matrix runner label = %q, want %q", tuple.OS, tuple.Runner, wantRunnerMarker[tuple.OS])
}
if tuple.Coverage || tuple.TestFlags != "" {
t.Errorf(
"runner %q has unexpected shared matrix fields: coverage=%t test-flags=%q",
tuple.OS,
tuple.Coverage,
tuple.TestFlags,
)
}
if len(tuple.Extra) != 0 {
t.Errorf("runner %q has unexpected matrix fields: %v", tuple.OS, tuple.Extra)
}
}
docStep := job.step(t, "Exercise native date and Bash process boundary")
const (
requiredSuiteSelector = "-required-suite=doc-freshness"
wantDocCommand = "go test '-tags=integration,gms_pure_go' -count=1 -run '^(TestDocFreshness.*|TestRequiredSuiteContract)$' ./scripts -args " + requiredSuiteSelector
)
if docStep.Run != wantDocCommand {
t.Errorf("doc-freshness command = %q, want required-suite execution %q", docStep.Run, wantDocCommand)
}
eolStep := job.step(t, "Exercise repository text EOL policy boundary")
const wantEOLCommand = "go test '-tags=integration,gms_pure_go' -count=1 ./scripts/gitattributespolicy -args -required-host -expected-goos '${{ matrix.expected_goos }}'"
if eolStep.Run != wantEOLCommand {
t.Errorf("repository EOL command = %q, want %q", eolStep.Run, wantEOLCommand)
}
if eolStep.If != "" {
t.Errorf("repository EOL step has conditional if = %q", eolStep.If)
}
if strings.Contains(eolStep.Run, "-run") {
t.Errorf("repository EOL step may not filter the narrow package: %q", eolStep.Run)
}
if job.stepIndex(t, "Exercise native date and Bash process boundary") >=
job.stepIndex(t, "Exercise repository text EOL policy boundary") {
t.Error("repository EOL step must remain separate and follow doc freshness")
}
gate := workflow.job(t, "ci-gate")
if !contains(gate.Needs, "check-doc-freshness-platforms") {
t.Errorf("ci-gate does not need check-doc-freshness-platforms: %v", gate.Needs)
}
gateEnv := gate.step(t, "Evaluate CI gate").Env
const gateKey = "CHECK_DOC_FRESHNESS_PLATFORMS"
if want := "${{ needs.check-doc-freshness-platforms.result }}"; gateEnv[gateKey] != want {
t.Errorf("ci-gate env %s = %q, want %q", gateKey, gateEnv[gateKey], want)
}
if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), gateKey) {
t.Errorf("ci-gate CI_GATE_REQUIRED does not include %q", gateKey)
}
}
func TestGoCacheOwnershipTopology(t *testing.T) {
workflows := map[string]ciWorkflow{
"main.yml": readCIWorkflow(t, "main.yml"),
"pr.yml": readCIWorkflow(t, "pr.yml"),
"pr-risk.yml": readCIWorkflow(t, "pr-risk.yml"),
}
for workflowName, workflow := range workflows {
assertPinnedGoCacheActions(t, workflowName, workflow)
}
t.Run("monolithic cache action is forbidden", func(t *testing.T) {
if !isGoCacheActionFamily(cacheMonolithicActionFamily) || !isForbiddenGoCacheActionFamily(cacheMonolithicActionFamily) {
t.Fatal("actions/cache must be recognized as a forbidden cache action family")
}
})
// Blacksmith macOS: restores what blacksmith-macos-go-build-cache's
// Blacksmith leg saves and owns only its race cache (day-bucketed, S7).
assertGoCacheInventory(t, workflows["main.yml"].job(t, "test"), []goCacheStep{
mainRestoreModuleCache(), mainRestoreBuildCacheDayBucketedIf("non-race", ""), mainRestoreBuildCacheDayBucketedIf("race", ""),
saveBuildCacheDayBucketedAfterFailureIf("race", ""),
})
assertGoCacheInventory(t, workflows["main.yml"].job(t, "test-windows"), []goCacheStep{
mainRestoreModuleCache(), mainRestoreBuildCacheDayBucketedIf("non-race", ""), saveModuleCache(), saveBuildCacheDayBucketed("non-race"),
})
// macOS saver for pr-preflight-platforms'/check-doc-freshness-platforms'
// macOS legs: its blacksmith venue leg seeds the same-repo path (and the
// `test` job), its github leg the fork path.
assertGoCacheInventory(t, workflows["main.yml"].job(t, "blacksmith-macos-go-build-cache"), []goCacheStep{
mainRestoreModuleCache(),
mainRestoreBuildCacheDayBucketedIf("non-race", ""),
saveModuleCacheAfterFailure(),
saveBuildCacheDayBucketedAfterFailureIf("non-race", ""),
})
assertGoCacheInventory(t, workflows["pr.yml"].job(t, "worktree-remove-windows"), []goCacheStep{
restoreModuleCache(), restoreBuildCache("non-race"),
})
// F7b review (S2): restore-only non-race Go build cache seeded by
// main.yml's test-windows Blacksmith leg (warm-non-race-cache.sh) on the
// exact same runner selection, targeting this job's own cold cmd/bd test
// compile (the reviewer's measured long pole here).
assertGoCacheInventory(t, workflows["pr.yml"].job(t, "test-windows-liveness"), []goCacheStep{
restoreModuleCache(), restoreBuildCache("non-race"),
})
assertGoCacheInventory(t, workflows["pr.yml"].job(t, "advisory-reports"), []goCacheStep{restoreModuleCache()})
// F7b review (S2): check-doc-freshness-platforms additionally restores a
// restore-only non-race Go build cache, seeded by the same push-to-main
// savers pr-preflight-platforms' leg restores below (see that job's own
// comment in pr.yml for the per-OS seeder mapping).
assertGoCacheInventory(t, workflows["pr.yml"].job(t, "check-doc-freshness-platforms"), []goCacheStep{
restoreModuleCache(), restoreBuildCache("non-race"),
})
// F7b review (S1): the Windows shard split this job's matrix once had
// (an extra `shard: 2` leg) was reverted: it measurably failed to help
// (shard 2 only ever held 15-25s of work while shard 1 stayed at
// 450-480s). The job is a single leg per OS again, so its module-cache
// restore is unconditional, same as every other three-host job; see S2
// above for the added non-race build-cache restore.
assertGoCacheInventory(t, workflows["pr.yml"].job(t, "pr-preflight-platforms"), []goCacheStep{
restoreModuleCache(), restoreBuildCache("non-race"),
})
assertNoUnmanagedGoCacheSteps(t, workflows, map[string]map[string]bool{
"main.yml": {
"test": true, "test-windows": true,
"blacksmith-macos-go-build-cache": true,
},
"pr.yml": {
"worktree-remove-windows": true,
"check-doc-freshness-platforms": true, "pr-preflight-platforms": true, "advisory-reports": true,
"test-windows-liveness": true,
},
"pr-risk.yml": {},
})
mainTest := workflows["main.yml"].job(t, "test")
assertStepsBefore(t, mainTest, []string{"Restore Go module cache"}, []string{"Build", "Test"})
assertStepsBefore(t, mainTest, []string{"Restore non-race Go build cache"}, []string{"Build"})
assertStepsBefore(t, mainTest, []string{"Restore race Go build cache"}, []string{"Test"})
assertStepsBefore(t, mainTest, []string{"Test"}, []string{"Save race Go build cache"})
mainWindows := workflows["main.yml"].job(t, "test-windows")
assertStepsBefore(t, mainWindows, []string{"Restore Go module cache", "Restore non-race Go build cache"}, []string{"Build (pure Go regex)"})
assertStepsBefore(t, mainWindows, []string{"Build (pure Go regex)", "Smoke test - version", "Smoke test - help"}, []string{"Save Go module cache", "Save non-race Go build cache"})
assertStepsBefore(t, workflows["pr.yml"].job(t, "worktree-remove-windows"),
[]string{"Restore Go module cache", "Restore non-race Go build cache"}, []string{"Run native Windows worktree removal boundary tests"})
assertStepsBefore(t, workflows["pr.yml"].job(t, "check-doc-freshness-platforms"),
[]string{"Restore Go module cache"}, []string{"Exercise native date and Bash process boundary"})
assertStepsBefore(t, workflows["pr.yml"].job(t, "pr-preflight-platforms"),
[]string{"Restore Go module cache"}, []string{"Exercise the real Bash process boundary", "Exercise test.sh prebuilt binary path"})
assertStepsBefore(t, workflows["pr.yml"].job(t, "advisory-reports"),
[]string{"Restore Go module cache"}, []string{"Type-check every module under examples/"})
assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Build", "non-race")
assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Test", "race")
assertGoCacheEnv(t, workflows["main.yml"].job(t, "test-windows"), "Build (pure Go regex)", "non-race")
assertGoCacheEnv(t, workflows["pr.yml"].job(t, "worktree-remove-windows"), "Run native Windows worktree removal boundary tests", "non-race")
for _, workflowName := range []string{"pr.yml", "pr-risk.yml"} {
for jobName, job := range workflows[workflowName].Jobs {
for _, step := range job.Steps {
family := actionFamily(step.Uses)
if family == cacheSaveActionFamily || isForbiddenGoCacheActionFamily(family) {
t.Errorf("%s job %q may not save cache in PR workflow", workflowName, jobName)
}
}
}
}
assertGoCacheWriter(t, workflows["main.yml"], "test", mainBlacksmithMacOSTestRunsOn, "Save race Go build cache", failureSurvivingCacheSaveCondition(cacheMissCondition(goBuildCacheRestoreID("race"))))
assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainBlacksmithWindows4vcpuRunsOn, "Save Go module cache", cacheMissCondition(goModuleCacheRestoreID))
assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainBlacksmithWindows4vcpuRunsOn, "Save non-race Go build cache", cacheMissCondition(goBuildCacheRestoreID("non-race")))
assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-macos-go-build-cache", mainBlacksmithMacOSGoBuildCacheRunsOn, "Save Go module cache", saveModuleCacheAfterFailure().ifCondition)
assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-macos-go-build-cache", mainBlacksmithMacOSGoBuildCacheRunsOn, "Save non-race Go build cache", saveBuildCacheAfterFailureIf("non-race", "").ifCondition)
for _, target := range []struct{ workflow, job string }{
{"main.yml", "test"},
{"main.yml", "test-windows"},
{"main.yml", "blacksmith-macos-go-build-cache"},
{"pr.yml", "worktree-remove-windows"},
{"pr.yml", "check-doc-freshness-platforms"},
{"pr.yml", "pr-preflight-platforms"},
{"pr.yml", "advisory-reports"},
} {
if got := workflows[target.workflow].job(t, target.job).step(t, "Set up Go").ID; got != "setup-go" {
t.Errorf("%s job %q setup-go id = %q, want setup-go", target.workflow, target.job, got)
}
}
}
func TestServerStorageShardScriptRunsPrebuiltBinaryFromPackageDir(t *testing.T) {
// pr4107_corruption_test.go and journal_scope_completeness_test.go use
// paths relative to internal/storage/dolt (e.g. ../schema/migrations,
// ../issueops). `go test` runs a package's tests with cwd = the package
// dir, so those resolve; a prebuilt test binary inherits the invoking
// shell's cwd instead. server-storage-test-shard.sh is invoked from the
// repo root (//internal/storage/dolt's server-storage variants),
// so the prebuilt-binary branch must cd into the package dir itself,
// immediately before exec -- any earlier and it breaks the repo-root-
// relative manifest/discovery above it; any later, or absent, and the 6
// relative-path tests fail with "open ../issueops: no such file or
// directory".
path := filepath.Join(sourceRepoRoot(t), ".github", "scripts", "server-storage-test-shard.sh")
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
lines := strings.Split(string(data), "\n")
indexOfContains := func(want string) int {
for i, line := range lines {
if strings.Contains(line, want) {
return i
}
}
return -1
}
indexOfExact := func(want string) int {
for i, line := range lines {
if strings.TrimSpace(line) == want {
return i
}
}
return -1
}
discoveryIndex := indexOfContains(`grep -rh '^func Test' internal/storage/dolt/*_test.go`)
if discoveryIndex < 0 {
t.Fatal("could not find repo-root-relative test-discovery grep line")
}
prebuiltBranchIndex := indexOfContains(`if [ -x "$STORAGE_BINARY" ]; then`)
if prebuiltBranchIndex < 0 {
t.Fatal("could not find prebuilt-binary branch")
}
execIndex := indexOfContains(`exec "$STORAGE_BINARY"`)
if execIndex < 0 {
t.Fatal("could not find prebuilt-binary exec line")
}
fallbackExecIndex := indexOfContains(`exec go test`)
if fallbackExecIndex < 0 {
t.Fatal("could not find go-test fallback exec line")
}
if fallbackExecIndex < execIndex {
t.Fatal("go-test fallback exec line appears before the prebuilt-binary exec line -- branch order assumption violated")
}
cdIndex := indexOfExact("cd internal/storage/dolt")
if cdIndex < 0 {
t.Fatal(`script does not "cd internal/storage/dolt" before running the prebuilt binary -- ` +
`relative-path tests (../schema/migrations, ../issueops) will fail when this script ` +
`is invoked from the repo root, as //internal/storage/dolt:dolt_server_full_test does`)
}
if cdIndex <= discoveryIndex {
t.Fatalf("cd internal/storage/dolt at line %d is at or before the repo-root-relative test "+
"discovery grep at line %d -- that discovery must still run from the repo root",
cdIndex+1, discoveryIndex+1)
}
if cdIndex <= prebuiltBranchIndex || cdIndex >= execIndex {
t.Fatalf("cd internal/storage/dolt at line %d must sit strictly between the prebuilt-binary "+
"branch at line %d and its exec at line %d", cdIndex+1, prebuiltBranchIndex+1, execIndex+1)
}
// The go-test fallback already scopes via the ./internal/storage/dolt/
// argument (cwd = repo root is fine for `go test`); it must not also cd.
for i := execIndex + 1; i <= fallbackExecIndex; i++ {
if strings.TrimSpace(lines[i]) == "cd internal/storage/dolt" {
t.Fatalf("unexpected cd internal/storage/dolt at line %d in the go-test fallback branch -- "+
"it already scopes via the ./internal/storage/dolt/ argument", i+1)
}
}
}
func assertNoUnmanagedGoCacheSteps(t *testing.T, workflows map[string]ciWorkflow, managed map[string]map[string]bool) {
t.Helper()
for workflowName, workflow := range workflows {
for jobName, job := range workflow.Jobs {
for _, step := range job.Steps {
family := actionFamily(step.Uses)
if isForbiddenGoCacheActionFamily(family) {
t.Errorf("%s job %q has forbidden monolithic cache step %q", workflowName, jobName, step.Name)
continue
}
if isGoCacheActionFamily(family) && !managed[workflowName][jobName] {
t.Errorf("%s job %q has unmanaged cache step %q", workflowName, jobName, step.Name)
}
}
}
}
}
const (
setupGoActionFamily = "actions/setup-go"
setupNodeActionFamily = "actions/setup-node"
setupPythonActionFamily = "actions/setup-python"
cacheMonolithicActionFamily = "actions/cache"
cacheRestoreActionFamily = "actions/cache/restore"
cacheSaveActionFamily = "actions/cache/save"
setupGoSHA = "b7ad1dad31e06c5925ef5d2fc7ad053ef454303e"
setupNodeSHA = "820762786026740c76f36085b0efc47a31fe5020"
setupPythonSHA = "5fda3b95a4ea91299a34e894583c3862153e4b97"
cacheSHA = "55cc8345863c7cc4c66a329aec7e433d2d1c52a9"
goCacheSchema = "v2"
goBaseTag = "gms_pure_go"
goModuleCachePath = "~/go/pkg/mod"
goModuleCacheRestoreID = "restore-go-module-cache"
macOSRunner = "macos-latest"
)
type goCacheStep struct {
name string
id string
family string
key string
restoreKeys string
path string
ifCondition string
}
func goModuleCacheKey() string {
return "beads-go-mod-" + goCacheSchema + "-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }}"
}
func goModuleCacheRestoreKeys() string {
return "beads-go-mod-" + goCacheSchema + "-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-"
}
func goBuildCachePrefix(profile string) string {
// The base tag identifies the cache topology. Go's content-addressed
// cache includes compiler options, so extra tags can safely share it.
return "beads-go-build-" + goCacheSchema + "-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-" + goBaseTag + "-" + profile + "-"
}
func goBuildCacheKey(profile string) string {
return goBuildCachePrefix(profile) + "${{ github.sha }}"
}
// goCacheDateSuffix is the key suffix for the day-bucketed caches saved by
// main.yml's push-to-main-only Blacksmith seeders (blacksmith-go-build-cache,
// test-windows): bounded by go.sum content plus a UTC calendar
// day, not by commit sha, so an ordinary day of push traffic cannot exhaust
// Blacksmith's 25 GB/week/repo cache quota the way a brand-new multi-GB entry
// per push did (F7b review fix S7; measured ~1.74-2.4GB/push under the old
// per-sha scheme). Each seeder's own "Compute cache date" step (id:
// cache-date) in main.yml computes the day half; restore-keys are unaffected,
// since they are already valid prefixes of this format too (they end right
// before where the sha/hash segment used to start). Pinned by
// TestBlacksmithSaverCacheKeysAreNotPerCommit.
func goCacheDateSuffix() string {
return "${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }}"
}
func goBuildCacheDayBucketedKey(profile string) string {
return goBuildCachePrefix(profile) + goCacheDateSuffix()
}
// mainRestoreBuildCacheDayBucketedIf is mainRestoreBuildCacheIf with a
// day-bucketed key (S7), for main.yml's blacksmith-go-build-cache and
// test-windows seeders.
func mainRestoreBuildCacheDayBucketedIf(profile, condition string) goCacheStep {
step := mainRestoreBuildCacheIf(profile, condition)
step.key = goBuildCacheDayBucketedKey(profile)
return step
}
// saveBuildCacheDayBucketedAfterFailureIf is saveBuildCacheAfterFailureIf
// with a day-bucketed key (S7), for main.yml's blacksmith-go-build-cache
// seeder.
func saveBuildCacheDayBucketedAfterFailureIf(profile, condition string) goCacheStep {
step := saveBuildCacheAfterFailureIf(profile, condition)
step.key = goBuildCacheDayBucketedKey(profile)
return step
}
// saveBuildCacheDayBucketed is saveBuildCache with a day-bucketed key (S7),
// for main.yml's test-windows seeder.
func saveBuildCacheDayBucketed(profile string) goCacheStep {
step := saveBuildCache(profile)
step.key = goBuildCacheDayBucketedKey(profile)
return step
}
func goBuildCachePath(profile string) string { return "${{ runner.temp }}/go-cache/" + profile }
func goBuildCacheRestoreID(profile string) string {
return "restore-" + profile + "-go-build-cache"
}
func cacheMissCondition(restoreID string) string {
return "steps." + restoreID + ".outputs.cache-hit != 'true'"
}
func combineConditions(conditions ...string) string {
var nonEmpty []string
for _, condition := range conditions {
if condition != "" {
nonEmpty = append(nonEmpty, condition)
}
}
return strings.Join(nonEmpty, " && ")
}
func failureSurvivingCacheSaveCondition(conditions ...string) string {
return "${{ !cancelled() && " + combineConditions(conditions...) + " }}"
}
func restoreModuleCache() goCacheStep {
return goCacheStep{name: "Restore Go module cache", family: cacheRestoreActionFamily, key: goModuleCacheKey(), restoreKeys: goModuleCacheRestoreKeys(), path: goModuleCachePath}
}
func mainRestoreModuleCache() goCacheStep {
step := restoreModuleCache()
step.id = goModuleCacheRestoreID
return step
}
func saveModuleCache() goCacheStep { return saveModuleCacheIf("") }
func saveModuleCacheIf(condition string) goCacheStep {
return goCacheStep{name: "Save Go module cache", family: cacheSaveActionFamily, key: goModuleCacheKey(), path: goModuleCachePath, ifCondition: combineConditions(condition, cacheMissCondition(goModuleCacheRestoreID))}
}
// saveModuleCacheAfterFailure is saveModuleCache with a failure-surviving
// condition, for saver jobs where a flaky mid-build step should not
// prevent the module cache fetched before it from being saved.
func saveModuleCacheAfterFailure() goCacheStep {
step := saveModuleCache()
step.ifCondition = failureSurvivingCacheSaveCondition(cacheMissCondition(goModuleCacheRestoreID))
return step
}
func restoreBuildCache(profile string) goCacheStep {
return restoreBuildCacheIf(profile, "")
}
func restoreBuildCacheIf(profile, condition string) goCacheStep {
return goCacheStep{name: "Restore " + profile + " Go build cache", family: cacheRestoreActionFamily, key: goBuildCacheKey(profile), restoreKeys: goBuildCachePrefix(profile), path: goBuildCachePath(profile), ifCondition: condition}
}
func mainRestoreBuildCache(profile string) goCacheStep {
return mainRestoreBuildCacheIf(profile, "")
}
func mainRestoreBuildCacheIf(profile, condition string) goCacheStep {
step := restoreBuildCacheIf(profile, condition)
step.id = goBuildCacheRestoreID(profile)
return step
}
func saveBuildCache(profile string) goCacheStep { return saveBuildCacheIf(profile, "") }
func saveBuildCacheIf(profile, condition string) goCacheStep {
return goCacheStep{name: "Save " + profile + " Go build cache", family: cacheSaveActionFamily, key: goBuildCacheKey(profile), path: goBuildCachePath(profile), ifCondition: combineConditions(condition, cacheMissCondition(goBuildCacheRestoreID(profile)))}
}
// saveBuildCacheAfterFailureIf is saveBuildCache with a failure-surviving
// condition, optionally on an arbitrary matrix condition, for main.yml's
// blacksmith-go-build-cache job (F7b), which gates its race/non-race saves on
// matrix.flavor instead of matrix.os.
func saveBuildCacheAfterFailureIf(profile, condition string) goCacheStep {
step := saveBuildCache(profile)
step.ifCondition = failureSurvivingCacheSaveCondition(condition, cacheMissCondition(goBuildCacheRestoreID(profile)))
return step
}
func assertGoCacheInventory(t *testing.T, job ciWorkflowJob, want []goCacheStep) {
t.Helper()
var got []ciWorkflowStep
for _, step := range job.Steps {
if isGoCacheActionFamily(actionFamily(step.Uses)) {
got = append(got, step)
}
}
if len(got) != len(want) {
t.Fatalf("cache steps = %d, want %d; got %+v", len(got), len(want), got)
}
for i, expected := range want {
step := got[i]
if step.Name != expected.name || step.ID != expected.id || actionFamily(step.Uses) != expected.family || step.With["key"] != expected.key || step.With["restore-keys"] != expected.restoreKeys || step.With["path"] != expected.path || step.If != expected.ifCondition {
t.Errorf("cache step %d = {name:%q id:%q family:%q key:%q restore-keys:%q path:%q if:%q}, want {name:%q id:%q family:%q key:%q restore-keys:%q path:%q if:%q}", i, step.Name, step.ID, actionFamily(step.Uses), step.With["key"], step.With["restore-keys"], step.With["path"], step.If, expected.name, expected.id, expected.family, expected.key, expected.restoreKeys, expected.path, expected.ifCondition)
}
}
}
func assertStepsBefore(t *testing.T, job ciWorkflowJob, before, after []string) {
t.Helper()
for _, beforeName := range before {
beforeIndex := job.stepIndex(t, beforeName)
for _, afterName := range after {
afterIndex := job.stepIndex(t, afterName)
if beforeIndex >= afterIndex {
t.Errorf("step %q index %d must precede %q index %d", beforeName, beforeIndex, afterName, afterIndex)
}
}
}
}
func assertGoCacheEnv(t *testing.T, job ciWorkflowJob, stepName, profile string) {
t.Helper()
if got := job.step(t, stepName).Env["GOCACHE"]; got != goBuildCachePath(profile) {
t.Errorf("step %q GOCACHE = %q, want %q", stepName, got, goBuildCachePath(profile))
}
}
func assertStepRunsExactly(t *testing.T, job ciWorkflowJob, stepName, want string) {
t.Helper()
if got := job.step(t, stepName).Run; got != want {
t.Errorf("step %q run = %q, want %q", stepName, got, want)
}
}
func assertStepEnvValue(t *testing.T, job ciWorkflowJob, stepName, key, want string) {
t.Helper()
if got := job.step(t, stepName).Env[key]; got != want {
t.Errorf("step %q env %s = %q, want %q", stepName, key, got, want)
}
}
// matrixIncludeHosts returns the sorted, de-duplicated set of matrix.include
// os values -- used by tests covering pr-preflight-platforms and
// check-doc-freshness-platforms now that F7b restructured them from a
// top-level os: list into an include:-only "matrix marker" shape.
func matrixIncludeHosts(include []ciWorkflowMatrixInclude) []string {
seen := make(map[string]bool)
var hosts []string
for _, tuple := range include {
if !seen[tuple.OS] {
seen[tuple.OS] = true
hosts = append(hosts, tuple.OS)
}
}
sort.Strings(hosts)
return hosts
}
func assertGoCacheWriter(t *testing.T, workflow ciWorkflow, wantJob, wantRunner, wantStep, wantCondition string) {
t.Helper()
var writers []string
for jobName, job := range workflow.Jobs {
for _, step := range job.Steps {
if job.RunsOn == wantRunner && actionFamily(step.Uses) == cacheSaveActionFamily && step.Name == wantStep && step.If == wantCondition {
writers = append(writers, jobName+"/"+step.Name)
}
}
}
want := wantJob + "/" + wantStep
if len(writers) != 1 || writers[0] != want {
t.Errorf("cache writer %q with if %q = %v, want exactly [%s]", wantStep, wantCondition, writers, want)
}
}
var actionPin = regexp.MustCompile(`^[0-9a-f]{40}$`)
func actionFamily(uses string) string {
family, _, found := strings.Cut(uses, "@")
if !found {
return uses
}
return family
}
func isGoCacheActionFamily(family string) bool {
return family == cacheMonolithicActionFamily || family == cacheRestoreActionFamily || family == cacheSaveActionFamily
}
func isForbiddenGoCacheActionFamily(family string) bool {
return family == cacheMonolithicActionFamily
}
func assertPinnedGoCacheActions(t *testing.T, workflowName string, workflow ciWorkflow) {
t.Helper()
allowed := map[string]string{
setupGoActionFamily: setupGoSHA,
cacheRestoreActionFamily: cacheSHA,
cacheSaveActionFamily: cacheSHA,
}
for jobName, job := range workflow.Jobs {
for _, step := range job.Steps {
family := actionFamily(step.Uses)
wantSHA, managed := allowed[family]
if isForbiddenGoCacheActionFamily(family) {
managed = true
}
if !managed {
continue
}
_, sha, found := strings.Cut(step.Uses, "@")
if !found || !actionPin.MatchString(sha) {
t.Errorf("%s job %q step %q action %q is not pinned to exactly 40 lowercase hex characters", workflowName, jobName, step.Name, step.Uses)
}
if isForbiddenGoCacheActionFamily(family) {
t.Errorf("%s job %q step %q uses forbidden monolithic action %q", workflowName, jobName, step.Name, family)
continue
}
if !found || !actionPin.MatchString(sha) {
continue
}
if sha != wantSHA {
t.Errorf("%s job %q step %q action %q has SHA %q, want released SHA %q", workflowName, jobName, step.Name, family, sha, wantSHA)
}
if family == setupGoActionFamily && step.With["cache"] != "false" {
t.Errorf("%s job %q setup-go cache = %q, want false", workflowName, jobName, step.With["cache"])
}
}
}
}
// TestWorkflowsInstallPinnedDolt keeps the Dolt CLI under test pinned. Every
// workflow used to install it by piping dolthub/dolt's releases/latest
// install.sh, so the binary under test changed whenever upstream published —
// including backports, which can move "latest" backwards. When Dolt 2.3.0
// landed it regressed CALL DOLT_RESET('--hard'): roughly one freshly created
// database in twenty comes up with the procedure permanently broken
// ("Error 1105 (HY000): context canceled"), which made
// TestFreshBootstrapHealIncarnation fail on a coin flip. See
// scripts/ci/install-dolt.sh for the per-version measurements. The CLI is now
// pinned to the same release as the container image, so the two halves of
// every server-mode test (the per-test sql-server doltserver.Start launches,
// and the shared container) can never drift apart.
func TestWorkflowsInstallPinnedDolt(t *testing.T) {
workflowDir := filepath.Join(sourceRepoRoot(t), ".github", "workflows")
entries, err := os.ReadDir(workflowDir)
if err != nil {
t.Fatal(err)
}
installers := 0
for _, entry := range entries {
if entry.IsDir() || filepath.Ext(entry.Name()) != ".yml" {
continue
}
path := filepath.Join(workflowDir, entry.Name())
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
body := string(data)
if strings.Contains(body, "dolt/releases/latest") {
t.Errorf("%s installs dolt from releases/latest; use ./scripts/ci/install-dolt.sh so the "+
"binary under test is pinned", entry.Name())
}
installers += strings.Count(body, "scripts/ci/install-dolt.sh")
}
if installers == 0 {
t.Fatal("no workflow installs dolt via scripts/ci/install-dolt.sh — the pin is not wired up")
}
}
// TestPinnedDoltCLIMatchesContainerImage keeps the CLI pin, the sql-server
// container pin and the hermetic Bazel dolt (tools/bazel/dolt.bzl) on the same
// Dolt release. Server-mode tests run both at once
// against the same databases; a drifting pair tests a combination no release
// ever shipped.
func TestPinnedDoltCLIMatchesContainerImage(t *testing.T) {
root := sourceRepoRoot(t)
installer, err := os.ReadFile(filepath.Join(root, "scripts", "ci", "install-dolt.sh"))
if err != nil {
t.Fatal(err)
}
cliVersion := captureOne(t, `(?m)^readonly version="([0-9]+\.[0-9]+\.[0-9]+)"$`, string(installer), "scripts/ci/install-dolt.sh")
common, err := os.ReadFile(filepath.Join(root, "internal", "testutil", "testdoltcommon.go"))
if err != nil {
t.Fatal(err)
}
imageVersion := captureOne(t, `dolthub/dolt-sql-server:([0-9]+\.[0-9]+\.[0-9]+)`, string(common), "testdoltcommon.go:DoltDockerImage")
pullScript, err := os.ReadFile(filepath.Join(root, "scripts", "ci", "pull-dolt-image.sh"))
if err != nil {
t.Fatal(err)
}
pullVersion := captureOne(t, `dolthub/dolt-sql-server:([0-9]+\.[0-9]+\.[0-9]+)`, string(pullScript), "scripts/ci/pull-dolt-image.sh")
bazelRule, err := os.ReadFile(filepath.Join(root, "tools", "bazel", "dolt.bzl"))
if err != nil {
t.Fatal(err)
}
bazelVersion := captureOne(t, `(?m)^DOLT_VERSION = "([0-9]+\.[0-9]+\.[0-9]+)"$`, string(bazelRule), "tools/bazel/dolt.bzl:DOLT_VERSION")
if cliVersion != imageVersion || cliVersion != pullVersion || cliVersion != bazelVersion {
t.Errorf("dolt pins disagree: CLI %s, DoltDockerImage %s, pull-dolt-image.sh %s, tools/bazel/dolt.bzl %s",
cliVersion, imageVersion, pullVersion, bazelVersion)
}
}
func captureOne(t *testing.T, pattern, body, source string) string {
t.Helper()
matches := regexp.MustCompile(pattern).FindStringSubmatch(body)
if matches == nil {
t.Fatalf("%s does not match %s", source, pattern)
}
return matches[1]
}
// F3: package-mcp and package-npm, the two lanes bazel-gate.sh does not
// know about at all - their skip reason is the caller's package-gates
// input, never the rbe job's mode.
var bazelPackageJobs = map[string]bool{bazelPackageMCPJobName: true, bazelPackageNPMJobName: true}
const (
uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"
downloadArtifactSHA = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c"
checkoutSHA = "3d3c42e5aac5ba805825da76410c181273ba90b1"
// Same SHA/comment already used in this repo for this action (rbe-
// prewarm's mint step, update-flake-lock.yml).
appTokenActionSHA = "bcd2ba49218906704ab6c1aa796996da409d3eb1"
bazelCacheKeyPrefix = "bazel-repo-v4-${{ runner.os }}-"
bazelCacheKey = bazelCacheKeyPrefix + "${{ hashFiles('.bazelversion', 'MODULE.bazel', 'MODULE.bazel.lock') }}"
bazelCachePath = "${{ runner.temp }}/bazel-ci-cache"
// Save only from a push to main that missed the exact key: the content is
// fixed by the key, so re-saving every push only churns the quota.
bazelCacheSaveIf = "${{ always() && github.event_name == 'push' && github.ref == 'refs/heads/main' && " +
"steps.bazel.outcome == 'success' && steps.bazel.outputs.cache-hit != 'true' }}"
)
// bazel.yml's jobs: the rbe job that decides the execution mode, the
// --config=ci lane, one job per CI job a Bazel config mirrors, and the two
// package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings.
var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltRaceJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName, bazelRRCSeedJob, bazelRRCVerifyJob}
// The lanes that only run remotely (skipped unless the rbe job chose remote);
// bazel-integration runs remotely or with the read-only cache (bazelIntegIf);
// every other lane also runs locally.
var bazelRemoteOnlyJobs = map[string]bool{bazelEmbedJobName: true, bazelProxiedJobName: true, bazelServerJobName: true}
// The rbe job's decision step reads exactly one secret, and only to test it
// for emptiness: its env value is a boolean, not the secret.
const (
bazelRBESecretPath = ".jobs." + bazelRBEJobName + ".steps[0].env.HAS_EXECUTOR"
bazelRBESecretValue = "${{ secrets.RBE_WEST_EXECUTOR != '' }}"
)
// rbe-prewarm reads its one app secret (the private key; the App's public
// client-id is a literal) in exactly two places, mirroring the rbe job's own
// emptiness-test/mint pattern above: an env boolean (never an `if:`) gating
// the mint step, and the mint step's own `with:`
// (the only step in bazel.yml allowed to read a secret outside a
// setup-bazel env, alongside bazelRBESecretPath/Value). Step indices are
// steps[0]=credential check, [1]=mint, [2]=dispatch, [3]=record result (no
// checkout step: B1, security review of bdef342d5).
const (
bazelRBEPrewarmHasAppPath = ".jobs." + bazelRBEPrewarmJobName + ".steps[0].env.HAS_POOL_APP"
bazelRBEPrewarmHasAppValue = "${{ secrets.RBE_POOL_APP_PRIVATE_KEY != '' }}"
bazelRBEPrewarmKeyPath = ".jobs." + bazelRBEPrewarmJobName + ".steps[1].with.private-key"
bazelRBEPrewarmKeyValue = "${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}"
// The job-level continue-on-error path itself: see
// TestBazelWorkflowJobsAndExecutionMode's comment on the same exception.
bazelRBEPrewarmContinueOnErrorPath = ".jobs." + bazelRBEPrewarmJobName + ".continue-on-error"
)
// rbe-ci-bep-analytics-design.md (S3) adds two steps to every lane job
// ("CI analytics summary" and "Upload CI analytics summary"): reporting
// only (the extracted summary feeds a follow-on analysis pipeline, nothing
// in bazel.yml or pr.yml reads its output), so both carry a deliberate
// step-level continue-on-error - the only step-level exception
// TestBazelWorkflowSecretsAndFailureSurface allows, matched below by job,
// step name and content (ciAnalyticsContinueOnErrorPaths) rather than by a
// single fixed path the way rbe-prewarm's job-level one is, since every
// lane job has its own copy of these two steps at a different index.
// ciAnalyticsContinueOnErrorPaths returns the walkYAML path of every CI
// analytics step's continue-on-error key, so
// TestBazelWorkflowSecretsAndFailureSurface can allow exactly those and
// nothing else. Scoped three ways, not just by step name: the job must be
// one of ciAnalyticsLaneJobs (ci_analytics_workflow_test.go), the summary
// step's run must actually invoke ci_analytics_extract.py, and the upload
// step's artifact name must start with "ci-analytics-" - so a future step
// that happens to share one of these two names, in a job this design never
// touched, can't ride along on the exception unnoticed.
func ciAnalyticsContinueOnErrorPaths(t *testing.T) map[string]bool {
t.Helper()
paths := map[string]bool{}
for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs {
if _, ok := ciAnalyticsLaneJobs[name]; !ok {
continue
}
for i, step := range job.Steps {
switch {
case step.Name == "CI analytics summary" && strings.Contains(step.Run, "ci_analytics_extract.py"):
case step.Name == "Upload CI analytics summary" && strings.HasPrefix(step.With["name"], "ci-analytics-"):
default:
continue
}
paths[fmt.Sprintf(".jobs.%s.steps[%d].continue-on-error", name, i)] = true
}
}
return paths
}
// rbe-fork: the lanes' setup-bazel env that asks it for a certificate (modes
// fork-ro and fork-rw only), and the mint's status URL the rbe job probes.
const (
bazelForkRemoteValue = "${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }}"
bazelMintStatusURL = "https://rbe-mint.ops.gascity.com:8444/v1/status?repo="
)
// The only triggers bazel.yml may have. pull_request_target (and
// workflow_run) would run with secrets in the context of fork PRs. PRs and
// merge groups reach it only through pr.yml's call, so it runs once per PR.
var bazelWorkflowTriggers = []string{"push", "workflow_call", "workflow_dispatch"}
// What pr.yml's ci-gate does with each bazel.yml lane (every job but the rbe
// job, whose rbe-mode output the gate reads): a gated lane has a
// CI_GATE_REQUIRED id read from its workflow_call output; an advisory lane is
// not part of pr.yml's call at all (the call's inputs turn it off, checked by
// TestBazelGateSimulation), for the recorded reason. A new job in bazel.yml
// must be added to one of the two (TestBazelLaneIsGatedAlongsideLegacy).
var bazelLaneGateIDs = map[string]string{
bazelJobName: "BAZEL_TEST",
bazelPureJobName: "BAZEL_PURE",
// Every release target cross-compiled, cgo off (and nogo over each).
bazelReleaseCrossJobName: "BAZEL_RELEASE_CROSS",
bazelEmbedJobName: "BAZEL_EMBEDDED",
bazelDoltJobName: "BAZEL_DOLTSERVER",
// Remote-only PR Risk tiers (fork and Dependabot PRs rely on
// pr-risk.yml's legacy jobs, like the embedded tier's).
bazelProxiedJobName: "BAZEL_PROXIED",
bazelServerJobName: "BAZEL_SERVER_STORAGE",
// main.yml's integration jobs, required on PRs although their legacy
// twins run only on push to main: remotely on same-repo PRs, locally
// with the read-only cache (mode cache) on fork and Dependabot PRs.
bazelIntegJobName: "BAZEL_INTEGRATION",
// F3: never skippable on pr.yml (package-gates is always "on" there), so
// bazel-gate.sh's skip list never names them.
bazelPackageMCPJobName: "PACKAGE_MCP",
bazelPackageNPMJobName: "PACKAGE_NPM",
}
// bazelFlagGate: a lane pr.yml's ci-gate requires only while a committed
// top-level pr.yml flag is "true" (the WINDOWS_PREBUILT_REQUIRED pattern):
// the lane runs in pr.yml's call and always reports, its id is read into the
// gate step's env but is never in the static CI_GATE_REQUIRED list, and the
// run script adds it when the flag is "true". Until then it is advisory, so
// it carries job-level continue-on-error (a failure must not turn the call's
// aggregate, BAZEL, red); once required, its own id still reddens the gate.
type bazelFlagGate struct {
id string // its CI_GATE_REQUIRED id
flag string // pr.yml's top-level env flag
want string // the flag's committed value: flipping it is a reviewed change
}
var bazelFlagGatedLanes = map[string]bazelFlagGate{
// The cmd/bd Dolt-server tier (the Dolt-gated cmd/bd tests no other lane
// runs), advisory until its first clean runs.
bazelCmdDoltJobName: {id: "BAZEL_CMD_DOLT", flag: "BAZEL_CMD_DOLT_REQUIRED", want: "true"},
}
// bazelFlagGateRun is the run-script fragment that adds a flag-gated lane's
// id to CI_GATE_REQUIRED.
func bazelFlagGateRun(g bazelFlagGate) string {
return "if [[ \"$" + g.flag + "\" == \"true\" ]]; then\n" +
" CI_GATE_REQUIRED=\"$CI_GATE_REQUIRED " + g.id + "\"\n" +
"fi\n"
}
// rbe-prewarm: not a build/test lane at all (no Bazel config mirrors it), so
// it has nothing for ci-gate to require; it is advisory by design
// (continue-on-error, best-effort pre-warm dispatch) and reports only so
// TestBazelLaneIsGatedAlongsideLegacy's generic shape check still applies.
var bazelAdvisoryLanes = map[string]string{
bazelRBEPrewarmJobName: "best-effort pre-warm dispatch to gastownhall/gascity; never gates or blocks a lane",
}
// bazel-integration's if: remote (modes remote, fork-ro, fork-rw: enabled),
// or local with the read-only cache (mode cache: fork and Dependabot PRs
// while rbe-fork is closed, whose only PR-time integration run it is; a
// warm run builds and tests only what the PR changed), never plain local
// (rbe=off: a cold tagged race build of //... on one runner); and off when a
// caller passes integration: "off" (pr.yml and bazel-farm.yml pass "on"). A
// string input: on push and dispatch it is null, and null != 'off', so the
// lane keeps running on main.
const bazelIntegIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }}"
// bazelServerStorageIf: like bazelIntegIf, bazel-server-storage honors the
// `integration` input (an explicit "off" skips it too, gastownhall/beads
// #7486 review), but unlike bazel-integration/bazel-cmd-dolt it is
// remote-only (wantRemoteOnlyIf), not mode cache -- there is no PR-time
// warm-cache run of this tier, only remote or nothing.
const bazelServerStorageIf = "${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }}"
// F3: package-mcp and package-npm's if. Unlike every other lane's, it does
// not read the rbe job's outputs at all - only the caller's package-gates
// input - so these two never skip because of execution mode, only because a
// caller left package-gates at its default "off".
const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}"
// F3: the package gates' runners (bazelPackageRunsOn, in
// ci_blacksmith_runner_test.go with the other runner sizes) are sized for
// pytest/npm on the runner itself, not for a Bazel client alone.
// Mode remote only: the gates take no rbe-fork certificate, so fork modes
// (enabled too) build bd with go build on a GitHub-hosted runner, as before.
// rbe-prewarm's bespoke if (TestBazelWorkflowJobsAndExecutionMode): mode
// remote only (B1, security review of bdef342d5 - was remote or fork-rw).
// fork-rw is reached only by a fork or Dependabot pull_request run, which
// never carries a workflow_call secret regardless of this if (GitHub does
// not forward repository/App secrets to a fork's pull_request event), so
// that half of the old condition always found no-app and no-op'd; it also
// cost an idle runner-minute for nothing. bazel-farm.yml no longer forwards
// the app secrets either (TestBazelFarmWorkflowSecurity), removing the only
// other path that could have reached fork-rw with the credential attached.
// Its runs-on still reuses the exact same mode-ternary every other lane
// uses (wantRunsOn): TestBazelRBEJobDecidesOnce forbids re-deriving
// fork/event facts outside the rbe job.
const bazelRBEPrewarmIf = "${{ needs.rbe.outputs.mode == 'remote' }}"
// F3: the rbe job's own runner (not gated by its own outputs - it decides
// them). Same-repo PRs and merge_group/push/dispatch/schedule (never forks)
// get Blacksmith; a pull_request_target farm run and any fork or Dependabot
// PR stay GitHub-hosted. Pinned verbatim by TestSameRepoBlacksmithRunners.
const wantRBERunsOn = "${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}"
// The sameRepoBlacksmith{2,4,8}vcpu consts and the real-expression evaluator
// that checks them now live in ci_blacksmith_runner_test.go (shared with
// F7c; see that file's doc comment for the API F7c rebases onto).
// F7c: used to define its own sameRepoBlacksmith4vcpu here (same expression
// as sameRepoBlacksmith2vcpu with the 4 vCPU label, for advisory jobs that
// compile Go). F7a independently defined the same const; both are now served
// by the single shared sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go.
// main.yml's push-to-main-only Windows and macOS savers, on the exact labels
// pr.yml's Windows and macOS legs run on (no GitHub-hosted leg is left to
// seed, ga-96smfk.22).
const mainBlacksmithWindows4vcpuRunsOn = blacksmithWindows4vcpuRunsOn
const mainBlacksmithMacOSGoBuildCacheRunsOn = "${{ '" + blacksmithMacOSLabel + "' }}"
// main.yml's push-to-main-only macOS test job, on Blacksmith macOS.
const mainBlacksmithMacOSTestRunsOn = "${{ '" + blacksmithMacOSLabel + "' }}"
// pr.yml's call of bazel.yml: exactly these inputs (review D1 v2 N3). An rbe
// override would put every PR in local mode and ungate the embedded tier
// while the gate stays self-consistent; integration: "off" would drop the
// required integration lane (explicit "on", so the pin and the gate
// simulation, which reads these inputs, do not depend on the default).
var bazelPRCallWith = map[string]string{
"build-artifact-name": "bazel-ci-build-artifacts",
"integration": "on",
// F3: only pr.yml opts in; bazel-farm.yml and nightly.yml keep the
// default "off" (TestBazelGateSimulation's other callers).
"package-gates": "on",
// Mirrors pr.yml's BAZEL_CMD_DOLT_REQUIRED (a `with:` cannot read env):
// true leaves //cmd/bd:bd_test to the then-required bazel-cmd-dolt
// (TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers).
"cmd-dolt-required": bazelFlagGatedLanes[bazelCmdDoltJobName].want,
}
// The call's aggregate result (needs.bazel.result, through bazel-gate.sh).
const bazelAggregateGateID = "BAZEL"
// The gate script and the rbe job's execution modes.
const bazelGateScript = ".github/scripts/bazel-gate.sh"
var bazelRBEModes = []string{"remote", "fork-ro", "fork-rw", "cache", "local", "skip"}
// bazelRemoteModes: the modes whose lanes execute on rbe-west (the rbe job's
// enabled output is "true" in exactly these): remote with the CI secrets,
// fork-ro and fork-rw with an rbe-fork-mint certificate.
var bazelRemoteModes = map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true}
// bazelModeEnabled: the rbe job's enabled output for a mode.
func bazelModeEnabled(mode string) string { return strconv.FormatBool(bazelRemoteModes[mode]) }
// The four RBE secrets plus the rbe-prewarm job's "bazel-allocator" GitHub
// App private key (it mints its own gastownhall/gascity installation
// token from it, inline in the job's own dispatch step - see bazel.yml's
// rbe-prewarm job), the full set bazel.yml declares under workflow_call and
// the only secrets pr.yml or nightly.yml may hand it
// (TestBazelLaneIsGatedAlongsideLegacy below).
var bazelCallSecrets = map[string]string{
"RBE_WEST_EXECUTOR": "${{ secrets.RBE_WEST_EXECUTOR }}",
"RBE_TLS_CERT": "${{ secrets.RBE_TLS_CERT }}",
"RBE_TLS_KEY": "${{ secrets.RBE_TLS_KEY }}",
"RBE_TLS_CA": "${{ secrets.RBE_TLS_CA }}",
"RBE_POOL_APP_PRIVATE_KEY": "${{ secrets.RBE_POOL_APP_PRIVATE_KEY }}",
}
// bazel-farm.yml's pull_request_target run must never carry the
// gastownhall/gascity app credential (B1, security review of bdef342d5): an
// allowlisted fork author's own PR code runs under that trust tier, and the
// app secrets mint a token with write access to another organization's
// repository. Only pr.yml and nightly.yml (same-repo/trusted triggers only)
// pass the RBE_POOL_APP_PRIVATE_KEY secret; bazel-farm.yml passes exactly these
// four.
var bazelFarmCallSecrets = map[string]string{
"RBE_WEST_EXECUTOR": "${{ secrets.RBE_WEST_EXECUTOR }}",
"RBE_TLS_CERT": "${{ secrets.RBE_TLS_CERT }}",
"RBE_TLS_KEY": "${{ secrets.RBE_TLS_KEY }}",
"RBE_TLS_CA": "${{ secrets.RBE_TLS_CA }}",
}
type ciCompositeAction struct {
Runs struct {
Using string `yaml:"using"`
Steps []ciWorkflowStep `yaml:"steps"`
} `yaml:"runs"`
}
func readSetupBazelAction(t *testing.T) ciCompositeAction {
t.Helper()
path := filepath.Join(sourceRepoRoot(t), setupBazelActionDir, "action.yml")
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var action ciCompositeAction
if err := yaml.Unmarshal(data, &action); err != nil {
t.Fatalf("parse %s: %v", path, err)
}
if action.Runs.Using != "composite" {
t.Fatalf("%s runs.using = %q, want composite", path, action.Runs.Using)
}
return action
}
// bazel.yml's jobs, their runner, rbe-west gate and skip rules, and the
// setup-bazel env. The skip rules are also what pr.yml's ci-gate accepts as a
// skip (TestBazelGateSimulation).
func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) {
workflow := readCIWorkflow(t, bazelWorkflowName)
var names []string
for name, job := range workflow.Jobs {
names = append(names, name)
// rbe-prewarm is the one deliberate exception: it is advisory (never
// a `needs` of any lane, never in ci-gate's required list - see
// bazelAdvisoryLanes and TestBazelLaneIsGatedAlongsideLegacy), and a
// called reusable workflow's overall conclusion is "failure" if ANY
// job within it fails regardless of `needs`, so without job-level
// continue-on-error here a pre-warm hiccup would still turn
// bazel.yml's own run - and thus pr.yml's ci-gate - red for a job
// nothing requires.
_, flagGated := bazelFlagGatedLanes[name]
// So is each shadow lane (bazelShadowLanes): it reports into no
// gate, so its failure must not fail the call (BAZEL) either.
_, shadow := bazelShadowLanes[name]
if job.ContinueOnError && name != bazelRBEPrewarmJobName && !flagGated && !shadow {
t.Errorf("%s continue-on-error hides failures from pr.yml's ci-gate", name)
}
if shadow && !job.ContinueOnError {
t.Errorf("%s is a shadow lane (%s) but lacks job-level continue-on-error", name, bazelShadowLanes[name])
}
// A flag-gated lane is advisory until its flag says otherwise, and
// an advisory job's failure must not fail the call (BAZEL).
if flagGated && !job.ContinueOnError {
t.Errorf("%s is flag-gated (advisory until pr.yml's %s is \"true\") but lacks job-level continue-on-error", name, bazelFlagGatedLanes[name].flag)
}
if job.TimeoutMinutes == 0 {
t.Errorf("%s has no timeout-minutes", name)
}
}
sort.Strings(names)
if !reflect.DeepEqual(names, bazelJobNames) {
t.Errorf("%s jobs = %v, want %v", bazelWorkflowName, names, bazelJobNames)
}
// Every lane needs the rbe job and takes its runner, skip rule and
// setup-bazel env from that job's outputs alone. Remote runs use the
// Blacksmith pool; fork modes (fork-ro/fork-rw: remote execution with an
// rbe-fork-mint certificate, the runner only a client), local and cache
// runs (forks while rbe-fork is closed, rbe=off/cache) the GitHub-hosted
// runner. Only mode remote gets the secrets; fork modes get
// BAZEL_FORK_REMOTE (setup-bazel mints the certificate), cache runs
// BAZEL_FORK_CACHE.
wantRunsOn := bazelLaneRunsOn
// Local-capable lanes skip only in mode skip (same-repo, RBE_WEST_WORKERS
// unset); remote-only lanes (27 race processes and more, an hour or more
// on a GitHub-hosted runner, for tests the Go jobs already run there)
// skip unless remote.
const wantIf = "${{ needs.rbe.outputs.mode != 'skip' }}"
const wantRemoteOnlyIf = "${{ needs.rbe.outputs.enabled == 'true' }}"
gate := "needs.rbe.outputs.mode == 'remote' && "
// The package gates' setup-bazel (mode remote only, see
// bazelPackageRunsOn): the secrets, never a fork certificate.
wantPackageSetupEnv := map[string]string{
"BAZEL_REMOTE_EXECUTOR": "${{ " + gate + "secrets.RBE_WEST_EXECUTOR || '' }}",
"RBE_TLS_CERT": "${{ " + gate + "secrets.RBE_TLS_CERT || '' }}",
"RBE_TLS_KEY": "${{ " + gate + "secrets.RBE_TLS_KEY || '' }}",
"RBE_TLS_CA": "${{ " + gate + "secrets.RBE_TLS_CA || '' }}",
"RBE_INSTANCE": "${{ " + gate + "'oss' || '' }}",
"BAZEL_FORK_CACHE": "${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }}",
}
wantSetupEnv := copyMap(wantPackageSetupEnv)
wantSetupEnv["BAZEL_FORK_REMOTE"] = bazelForkRemoteValue
wantSetupEnv["RBE_FORK_TIER"] = "${{ needs.rbe.outputs.tier }}"
wantSetupEnv["RBE_FORK_PR"] = "${{ github.event.pull_request.number }}"
// F3: the rbe job itself runs on Blacksmith for same-repo PRs (and
// merge_group/push/dispatch/schedule, which are never forks); a
// pull_request_target farm run and any fork or Dependabot PR stay
// GitHub-hosted. Pinned again, verbatim, by TestSameRepoBlacksmithRunners.
if rbe, ok := workflow.Jobs[bazelRBEJobName]; !ok {
t.Fatalf("%s has no %s job", bazelWorkflowName, bazelRBEJobName)
} else if rbe.RunsOn != wantRBERunsOn {
t.Errorf("%s runs-on = %q, want %q", bazelRBEJobName, rbe.RunsOn, wantRBERunsOn)
}
for name, job := range workflow.Jobs {
if name == bazelRBEJobName {
continue
}
if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) {
t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName)
}
if isBazelRRCJob(name) {
continue // runner, if and setup-bazel: bazel_rrc_test.go
}
// F3: the package gates use larger runners (bazelPackageRunsOn) and
// their own if (the caller's package-gates input, not the rbe job's
// mode - they never skip for execution-mode reasons).
wantJobRunsOn, wantJobIf, wantJobSetupEnv := wantRunsOn, wantIf, wantSetupEnv
if name == bazelJobName {
wantJobRunsOn = bazelTestLaneRunsOn
}
// The remote repo contents cache readers opt in to reading it in
// mode cache (BAZEL_RRC_READ, bazel_rrc_test.go); those that run
// there fall back to ubuntu-24.04.
if slices.Contains(bazelRRCReadLanes, name) {
wantJobSetupEnv = copyMap(wantSetupEnv)
wantJobSetupEnv["BAZEL_RRC_READ"] = "true"
}
if isBazelRRCCacheReader(name) {
wantJobRunsOn = bazelRRCCacheReaderRunsOn("blacksmith-4vcpu-ubuntu-2404")
if name == bazelJobName {
wantJobRunsOn = bazelRRCCacheReaderRunsOn("blacksmith-8vcpu-ubuntu-2404")
}
}
if bazelPackageJobs[name] {
wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn[name], bazelPackageGatesIf, wantPackageSetupEnv
} else if bazelRemoteOnlyJobs[name] {
wantJobIf = wantRemoteOnlyIf
if name == bazelServerJobName {
// Remote-only like bazel-embedded/bazel-proxied, but also
// honors the `integration` input like
// bazel-integration/bazel-cmd-dolt (gastownhall/beads#7486
// review).
wantJobIf = bazelServerStorageIf
}
} else if name == bazelIntegJobName || name == bazelCmdDoltJobName {
// The cmd/bd Dolt-server tier runs exactly where the integration
// lane does: its build is --config=integration's.
wantJobIf = bazelIntegIf
} else if name == bazelRBEPrewarmJobName {
// The lanes' runs-on ternary at 2 vCPU (no Bazel client); only
// the size and the if differ. Not a lane: never uses
// setup-bazel, so wantJobSetupEnv is moot (the per-step check
// below only fires for a setup-bazel step).
wantJobRunsOn, wantJobIf = bazelPrewarmRunsOn, bazelRBEPrewarmIf
} else if name == bazelDoltRaceJobName {
// Mode remote, on pull_request and merge_group events only
// (bazel_dolt_race_shadow_test.go); the lanes' runner and
// setup-bazel env otherwise.
wantJobIf = bazelDoltRaceJobIf
}
if job.RunsOn != wantJobRunsOn {
t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantJobRunsOn)
}
if job.If != wantJobIf {
t.Errorf("%s if = %q, want %q", name, job.If, wantJobIf)
}
for _, step := range job.Steps {
if step.Uses == "./"+setupBazelActionDir && !reflect.DeepEqual(step.Env, wantJobSetupEnv) {
t.Errorf("%s setup-bazel env = %v, want %v", name, step.Env, wantJobSetupEnv)
}
}
}
}
// F3: bazelPackageRunsOn's comment and the package-mcp job's runner size
// are both premised on mcp_pytest() in package-mcp.sh passing an explicit
// worker count (not "-n auto", which would reintroduce the CPU-count/
// cgroup-quota mismatch the explicit count was pinned to avoid): the
// default 8, or BEADS_MCP_PYTEST_WORKERS, which bazel.yml's package-mcp sets
// to 16 exactly when it runs on its 8 vCPU Blacksmith runner
// (TestBlacksmithBazelRunnerSizes). It is also premised on pytest-xdist
// itself being a locked dev dependency so that worker count is reproducible
// across CI runs. Pin both here directly against the script and the MCP
// package's manifest/lock.
func TestMCPPytestWorkerCountPinned(t *testing.T) {
root := sourceRepoRoot(t)
script := readPolicyFile(t, root, "scripts/ci/package-mcp.sh")
for _, want := range []string{`local workers="${BEADS_MCP_PYTEST_WORKERS:-8}"`, `uv run pytest -n "$workers"`} {
if !strings.Contains(script, want) {
t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() is missing %q (an explicit worker count, default 8)", want)
}
}
if strings.Contains(script, "-n auto") {
t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() uses -n auto; want an explicit worker count")
}
pyproject := readPolicyFile(t, root, "integrations/beads-mcp/pyproject.toml")
if !regexp.MustCompile(`(?m)^\s*"pytest-xdist[><=]`).MatchString(pyproject) {
t.Errorf("integrations/beads-mcp/pyproject.toml does not pin pytest-xdist as a dev dependency")
}
lock := readPolicyFile(t, root, "integrations/beads-mcp/uv.lock")
if !strings.Contains(lock, "name = \"pytest-xdist\"") {
t.Errorf("integrations/beads-mcp/uv.lock has no locked pytest-xdist entry; run uv lock")
}
}
// F3: package-mcp and package-npm's job definitions (including the very
// detect step this script's output feeds) moved into bazel.yml, so a PR that
// edits only bazel.yml's package-gate logic must still self-trigger both
// gates the same way editing pr.yml/main.yml/pr-risk.yml does; otherwise a
// change to the job that runs the gates could ship without the gates having
// run against it. No other policy test reads this script's content.
func TestDetectPackageGatesCoversBazelWorkflow(t *testing.T) {
root := sourceRepoRoot(t)
script := readPolicyFile(t, root, "scripts/ci/detect-package-gates.sh")
re := regexp.MustCompile(`(?s)scripts/ci/detect-package-gates\.sh\|[^)]*\)\s*\n\s*mcp_package=true\s*\n\s*npm_package=true`)
loc := re.FindString(script)
if loc == "" {
t.Fatalf("scripts/ci/detect-package-gates.sh has no case arm forcing both mcp_package and npm_package true alongside its own path")
}
if !strings.Contains(loc, ".github/workflows/bazel.yml") {
t.Errorf("scripts/ci/detect-package-gates.sh's both-gates case arm does not include .github/workflows/bazel.yml, where the package-mcp/package-npm job definitions now live")
}
// F3 review NIT-6: package-bazel-bd.sh packages the Bazel-built bd for
// both package-mcp and package-npm on the remote-execution path, so a
// change to it must run both gates too.
if !strings.Contains(loc, "scripts/ci/package-bazel-bd.sh") {
t.Errorf("scripts/ci/detect-package-gates.sh's both-gates case arm does not include scripts/ci/package-bazel-bd.sh, which both package gates use to package the Bazel-built bd")
}
}
// F3: the exact "same-repo PR, or merge_group" Blacksmith expression, pinned
// once per job so a future edit cannot drift one copy from another. Reads no
// repo variable, so the rollback is a plain revert. The rbe job's own
// version additionally covers push, workflow_dispatch and schedule (never
// forks); every other job below reads only merge_group and pull_request.
func TestSameRepoBlacksmithRunners(t *testing.T) {
want := map[string]map[string]string{
// F7a: cache-free same-repo Linux jobs that restore no GitHub-saved
// build cache moved onto same-repo Blacksmith at their measured vCPU
// size (module-cache misses on the Blacksmith pool are acceptable
// per spec).
"pr.yml": {
"ci-gate": sameRepoBlacksmith2vcpu,
"fast-checks": sameRepoBlacksmith2vcpu,
"advisory-reports": sameRepoBlacksmith2vcpu,
"check-doc-flags": sameRepoBlacksmith4vcpu,
// Windows and macOS: Blacksmith for every PR, forks and Dependabot
// included (ga-96smfk.22). test-windows-liveness and
// worktree-remove-windows are sized at 4vcpu (a build plus a
// focused test run each); test-windows-small at 2vcpu;
// windows-make-shell at 4vcpu (installs three separate Make
// toolchains).
"test-windows-liveness": blacksmithWindows4vcpuRunsOn,
"worktree-remove-windows": blacksmithWindows4vcpuRunsOn,
"test-windows-small": blacksmithWindows2vcpuRunsOn,
"windows-make-shell": blacksmithWindows4vcpuRunsOn,
// Mixed-OS matrix jobs: each leg names its Blacksmith label
// (platformsMatrixRunners, pinned by their own tests).
"check-doc-freshness-platforms": platformsMatrixRunsOn,
"pr-preflight-platforms": platformsMatrixRunsOn,
},
"pr-risk.yml": {
"ci-gate": sameRepoBlacksmith2vcpu,
"test-nix": sameRepoBlacksmith4vcpu,
},
bazelWorkflowName: {bazelRBEJobName: wantRBERunsOn},
// F7c: advisory workflows. Each compiles Go (or, for broken-links,
// is cheap enough to size at 2 vCPU per spec-f7.md §2.2) and moves
// to Blacksmith for same-repo PRs/merge_group only; forks,
// Dependabot and push stay on ubuntu-latest.
"docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu},
// main.yml's push-to-main-only jobs. Tracked here so the "no other
// Blacksmith label leaks" sweep below also covers main.yml.
"main.yml": {
// F7b review (B2): these push-to-main-only jobs seed the
// Blacksmith-selection GOCACHEs the PR jobs above restore from.
// Each is additionally guarded by a job-level
// github.event_name == 'push' && github.ref ==
// 'refs/heads/main' && github.repository == 'gastownhall/beads'
// if: (TestBlacksmithSaverJobsGuardedAgainstPullRequest below),
// matching F7c's own saver convention.
"test-windows": mainBlacksmithWindows4vcpuRunsOn,
// Same guard, seeding pr.yml's macOS legs.
"blacksmith-macos-go-build-cache": mainBlacksmithMacOSGoBuildCacheRunsOn,
// The macOS -race -short suite (Linux runs it under Bazel).
"test": mainBlacksmithMacOSTestRunsOn,
},
}
// The two required gates' display names are a stable external contract
// (branch protection rule names) - moving them to Blacksmith must not
// rename them. Reviewed 2026-10-03: a rename of either survived every
// other policy test, since nothing else reads job.Name.
wantGateName := map[string]map[string]string{
"pr.yml": {"ci-gate": "CI Gate / Required"},
"pr-risk.yml": {"ci-gate": "PR Risk Gate / Required"},
}
for file, jobs := range want {
workflow := readCIWorkflow(t, file)
for name, wantRunsOn := range jobs {
job, ok := workflow.Jobs[name]
if !ok {
t.Fatalf("%s has no %s job", file, name)
}
if job.RunsOn != wantRunsOn {
t.Errorf("%s %s runs-on = %q, want %q", file, name, job.RunsOn, wantRunsOn)
}
if wantName := wantGateName[file][name]; wantName != "" && job.Name != wantName {
t.Errorf("%s %s name = %q, want %q", file, name, job.Name, wantName)
}
// F3: every job moved onto Blacksmith needs its own
// timeout-minutes - a hang there burns paid Blacksmith minutes,
// not just a free GitHub-hosted runner's default 360.
if job.TimeoutMinutes == 0 {
t.Errorf("%s %s has no timeout-minutes (now runs on Blacksmith for same-repo PRs)", file, name)
}
}
}
// No other job in pr.yml or pr-risk.yml may name a Blacksmith label beyond
// the ones enumerated in `want` above.
// Same sweep for the F7c advisory workflows: nothing in them may name a
// Blacksmith label beyond the jobs listed in `want` above.
for _, file := range []string{
"pr.yml", "pr-risk.yml", "main.yml",
"docs-mintlify.yml",
} {
workflow := readCIWorkflow(t, file)
allowed := want[file]
for name, job := range workflow.Jobs {
if allowed[name] != "" {
continue
}
if strings.Contains(job.RunsOn, "blacksmith-") {
t.Errorf("%s job %s runs-on %q names a Blacksmith label; only %v may", file, name, job.RunsOn, allowed)
}
}
}
}
// TestBlacksmithJobsReadNoSecrets: F7a's rule that a job whose runs-on can
// select a Blacksmith label never reads a secret, in any form (job env, job
// if:, step env, step with:, step if:, or a step's run: body interpolating
// ${{ secrets.X }}). Blacksmith's ternary already falls back to ubuntu-latest
// for forks and Dependabot, but the no-secrets rule is a second, independent
// backstop: even a same-repo Blacksmith run should never need repository
// secrets for a cache-free, read-only check.
//
// Review N-4 (2026-10-03): this also now scans job.If and step.If, since a
// conditional guarded on a secret (e.g. an if: expression testing
// secrets.FOO) would otherwise slip past the env/with/run checks below. It
// does not yet inspect
// container/services credentials or resolve matrix markers in runs-on
// (neither ciWorkflowJob field exists today); that is deferred to F7b, which
// is where container/services and matrix-aware runs-on resolution land.
func TestBlacksmithJobsReadNoSecrets(t *testing.T) {
secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`)
for _, file := range []string{"pr.yml", "pr-risk.yml"} {
workflow := readCIWorkflow(t, file)
for name, job := range workflow.Jobs {
if !strings.Contains(job.RunsOn, "blacksmith-") {
continue
}
if secretRef.MatchString(job.If) {
t.Errorf("%s job %s if %q reads a secret; Blacksmith-eligible jobs must not", file, name, job.If)
}
for k, v := range job.Env {
if secretRef.MatchString(v) {
t.Errorf("%s job %s env %s = %q reads a secret; Blacksmith-eligible jobs must not", file, name, k, v)
}
}
for _, step := range job.Steps {
if secretRef.MatchString(step.If) {
t.Errorf("%s job %s step %q if %q reads a secret", file, name, step.Name, step.If)
}
for k, v := range step.Env {
if secretRef.MatchString(v) {
t.Errorf("%s job %s step %q env %s = %q reads a secret", file, name, step.Name, k, v)
}
}
for k, v := range step.With {
if secretRef.MatchString(v) {
t.Errorf("%s job %s step %q with %s = %q reads a secret", file, name, step.Name, k, v)
}
}
if secretRef.MatchString(step.Run) {
t.Errorf("%s job %s step %q run reads a secret:\n%s", file, name, step.Name, step.Run)
}
}
}
}
}
// TestSameRepoBlacksmithExpressionSemantics now lives in
// ci_blacksmith_runner_test.go, which evaluates the real
// sameRepoBlacksmith{2,4,8}vcpu expression strings with a small GitHub
// Actions expression evaluator instead of a hand-written Go mirror
// (review SF-3). That file is shared with F7c.
// TestPRCIGateFastChecksTokens pins pr.yml's fast-checks job, the base-diff
// checks no Bazel target can hold (check-migration-hygiene.sh's delta half
// and the .beads/issues.jsonl guard): each step's id, its outputs entry, and
// the CI_GATE_REQUIRED token/env mapping, plus the FAST_CHECKS backstop
// token (the job's own .result), which gives ci-gate.sh one clear line if the
// whole job is lost before any step reports. The job's former tree checks
// are Bazel tests (TestPRPolicyChecksRunAsBazelTargets); their tokens must
// not linger.
func TestPRCIGateFastChecksTokens(t *testing.T) {
pr := readCIWorkflow(t, "pr.yml")
job := pr.job(t, "fast-checks")
gate := pr.job(t, "ci-gate")
gateStep := gate.step(t, "Evaluate CI gate")
required := strings.Fields(gateStep.Env["CI_GATE_REQUIRED"])
if !contains(gate.Needs, "fast-checks") {
t.Errorf("ci-gate does not need fast-checks: %v", gate.Needs)
}
if !contains(required, "FAST_CHECKS") || gateStep.Env["FAST_CHECKS"] != "${{ needs.fast-checks.result }}" {
t.Errorf("ci-gate does not require fast-checks' own result as the FAST_CHECKS backstop: required=%v env=%q", required, gateStep.Env["FAST_CHECKS"])
}
// stepID -> (step name, job output name, CI_GATE_REQUIRED token).
mapping := map[string]struct{ name, output, token string }{
"migration-hygiene": {"Run migration hygiene checks", "migration-hygiene", "CHECK_MIGRATION_HYGIENE"},
"no-beads-changes": {"Check for .beads/issues.jsonl changes", "no-beads-changes", "CHECK_NO_BEADS_CHANGES"},
}
if len(job.Steps) != len(mapping)+1 {
t.Errorf("fast-checks has %d steps, want checkout plus %d: a tree-only check belongs in a Bazel target", len(job.Steps), len(mapping))
}
if len(job.Outputs) != len(mapping) {
t.Errorf("fast-checks outputs = %v, want exactly %d", job.Outputs, len(mapping))
}
for stepID, m := range mapping {
step := job.step(t, m.name)
// Both steps run on both PR triggers, merge_group included (the
// diffs take merge_group.base_sha there), so the gate excuses none
// of their skips.
wantIf := "${{ !cancelled() }}"
if step.ID != stepID || step.If != wantIf {
t.Errorf("fast-checks step %q: id %q, if %q; want id %q, if %q", step.Name, step.ID, step.If, stepID, wantIf)
}
wantOutput := "${{ steps." + stepID + ".outcome }}"
if job.Outputs[m.output] != wantOutput {
t.Errorf("fast-checks outputs.%s = %q, want %q", m.output, job.Outputs[m.output], wantOutput)
}
if !contains(required, m.token) {
t.Errorf("ci-gate CI_GATE_REQUIRED does not include %s", m.token)
}
wantEnv := "${{ needs.fast-checks.outputs." + m.output + " || 'skipped' }}"
if gateStep.Env[m.token] != wantEnv {
t.Errorf("ci-gate env %s = %q, want %q", m.token, gateStep.Env[m.token], wantEnv)
}
}
for _, retired := range []string{"CHECK_BUILD_TAGS", "CHECK_VERSION_CONSISTENCY", "FMT_CHECK"} {
if contains(required, retired) || gateStep.Env[retired] != "" {
t.Errorf("ci-gate still carries %s; that check is a Bazel test now", retired)
}
}
// Merge queue: the .beads guard runs on merge_group too, diffing
// against the queue's base commit (a merge group has no base_ref), and
// the gate no longer excuses its skip there.
noBeads := job.step(t, "Check for .beads/issues.jsonl changes")
if noBeads.Env["MERGE_GROUP_BASE_SHA"] != "${{ github.event.merge_group.base_sha }}" ||
!strings.Contains(noBeads.Run, `base="${MERGE_GROUP_BASE_SHA:-origin/${GITHUB_BASE_REF:-main}}"`) ||
!strings.Contains(noBeads.Run, `git diff --name-only "$base"...HEAD`) || strings.Contains(noBeads.Run, "${{") {
t.Errorf("fast-checks .beads guard must diff merge_group.base_sha (merge queue) or origin/$GITHUB_BASE_REF (pull_request) without interpolating expressions: env %v\n%s", noBeads.Env, noBeads.Run)
}
if strings.Contains(gateStep.Run, "CHECK_NO_BEADS_CHANGES") {
t.Error("ci-gate run script excuses a CHECK_NO_BEADS_CHANGES skip; the guard runs on merge_group too")
}
// The migration step runs only the delta half (checks C, D and E) of
// check-migration-hygiene.sh against the PR's base; the tree half is
// //scripts/repochecks:migration_hygiene_test.
migrationHygiene := job.step(t, "Run migration hygiene checks")
const wantBaseSHA = "${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}"
if migrationHygiene.Env["BASE_SHA"] != wantBaseSHA {
t.Errorf("fast-checks migration-hygiene step env BASE_SHA = %q, want %q", migrationHygiene.Env["BASE_SHA"], wantBaseSHA)
}
if migrationHygiene.Env["MIGRATION_HYGIENE_SCOPE"] != "delta" {
t.Errorf("fast-checks migration-hygiene step MIGRATION_HYGIENE_SCOPE = %q, want delta", migrationHygiene.Env["MIGRATION_HYGIENE_SCOPE"])
}
if migrationHygiene.Run != "./scripts/check-migration-hygiene.sh" {
t.Errorf("fast-checks migration-hygiene step run = %q", migrationHygiene.Run)
}
}
// TestPRPolicyChecksRunAsBazelTargets: every repository guard pr.yml's
// fast-checks, PR Policy and doc-freshness jobs used to run on a GitHub
// runner is a Bazel test (ga-96smfk.20), with no tag, so bazel.yml's `test`
// lane (--config=ci, remote) runs it and CI Gate / Required gates on it, or,
// for the integration-tagged doc-freshness suite, the integration lane.
func TestPRPolicyChecksRunAsBazelTargets(t *testing.T) {
root := bazelPolicyRoot(t)
for _, c := range []struct {
build, target, runs string
tags string
}{
{"scripts/repochecks/BUILD.bazel", "build_tags_test", `"scripts/check-build-tags.sh"`, ""},
{"scripts/repochecks/BUILD.bazel", "workflow_tags_test", `"$(rlocationpath //scripts/checkworkflowtags)"`, ""},
{"scripts/repochecks/BUILD.bazel", "go_install_guidance_test", `"scripts/check-go-install-guidance.sh"`, ""},
{"scripts/repochecks/BUILD.bazel", "winget_portable_alias_test", `"scripts/check-winget-portable-alias.sh"`, ""},
{"scripts/repochecks/BUILD.bazel", "migration_hygiene_test", `"MIGRATION_HYGIENE_SCOPE": "tree"`, ""},
{"scripts/repochecks/BUILD.bazel", "fmt_test", `"scripts/ci/fmt-check.sh"`, ""},
{"scripts/repochecks/BUILD.bazel", "doc_freshness_test", `"scripts/check-doc-freshness.sh"`, ""},
{"scripts/repochecks/BUILD.bazel", "workapi_frontend_boundary_test", `"scripts/check-workapi-frontend-boundary.sh"`, ""},
{"scripts/repochecks/BUILD.bazel", "types_gen_drift_test", `"types_gen_drift_test.sh"`, ""},
{"internal/versioncheck/BUILD.bazel", "versioncheck_test", `"//:release_metadata_files"`, ""},
{"scripts/BUILD.bazel", "go_sources_test", `"testing_short_tree_test.go"`, ""},
{"scripts/BUILD.bazel", "doc_freshness_required_test", `"-required-suite=doc-freshness"`, `["integration-only"]`},
{"scripts/gitattributespolicy/BUILD.bazel", "gitattributespolicy_required_host_test", `"-required-host"`, ""},
} {
var rule string
for _, r := range bazelTopRules(readPolicyFile(t, root, c.build)) {
if strings.Contains(r, "\n name = \""+c.target+"\",\n") {
rule = r
}
}
if rule == "" {
t.Errorf("%s: no target %s", c.build, c.target)
continue
}
if !strings.Contains(rule, c.runs) {
t.Errorf("%s:%s no longer names %s", c.build, c.target, c.runs)
}
tags := regexp.MustCompile(`(?m)^ tags = (\[.*\]),$`).FindStringSubmatch(rule)
switch {
case c.tags == "" && tags != nil:
t.Errorf("%s:%s is tagged %s; untagged, it runs in the test lane", c.build, c.target, tags[1])
case c.tags != "" && (tags == nil || tags[1] != c.tags):
t.Errorf("%s:%s tags = %v, want %s", c.build, c.target, tags, c.tags)
}
}
// main.yml (push to main) too: bazel.yml's push run gates the same
// targets, so no runner re-runs them after the merge.
for _, wfName := range []string{"pr.yml", "main.yml"} {
for name, job := range readCIWorkflow(t, wfName).Jobs {
for _, step := range job.Steps {
for _, moved := range []string{
"check-build-tags.sh", "check-go-install-guidance.sh", "check-winget-portable-alias.sh",
"check-versions.sh", "make fmt-check", "check-doc-freshness.sh", "make ci-pr-policy", "make api-check",
"check-migration-hygiene.sh",
} {
if strings.Contains(step.Run, moved) && !(wfName == "pr.yml" && moved == "check-migration-hygiene.sh") {
t.Errorf("%s job %s step %q still runs %s, which is a Bazel test", wfName, name, step.Name, moved)
}
}
}
}
}
pr := readCIWorkflow(t, "pr.yml")
freshness := pr.job(t, "check-doc-freshness-platforms")
for _, tuple := range freshness.Strategy.Matrix.Include {
if tuple.ExpectedGOOS == "linux" {
t.Errorf("check-doc-freshness-platforms still has a Linux leg (%s); Bazel runs it", tuple.OS)
}
}
}
// Review N-6 (2026-10-03): test-windows-small's dbproxy-server leg must keep
// building CGO-disabled, same as before the fold.
func TestPRWindowsSmallDbproxyServerIsCGODisabled(t *testing.T) {
pr := readCIWorkflow(t, "pr.yml")
job := pr.job(t, "test-windows-small")
step := job.step(t, "Run dbproxy server package tests on Windows")
if step.Env["CGO_ENABLED"] != "0" {
t.Errorf("test-windows-small dbproxy-server step env CGO_ENABLED = %q, want \"0\"", step.Env["CGO_ENABLED"])
}
}
// stepContinueOnError reports whether a step's continue-on-error is truthy
// (the field unmarshals as `any` because the YAML schema also allows an
// expression string, which GitHub itself treats as whatever it evaluates to
// - this test only cares about the literal `true` every current step uses).
func stepContinueOnError(step ciWorkflowStep) bool {
v, ok := step.ContinueOnError.(bool)
return ok && v
}
// TestPRWindowsMakeShellLegsFailTheJob is review SF-1 (2026-10-03):
// windows-make-shell is a required gate (WINDOWS_MAKE_SHELL). Before this
// fix, every leg's install/exercise step carried continue-on-error and a
// hand-written final step reconstructed the job's pass/fail from each step's
// .outcome - two mutations (an aggregator that always exits 0, and one that
// silently dropped the MSYS2 install check) both passed the suite, because
// nothing pinned the aggregator's own correctness. The fix removes the
// aggregator entirely: every step after checkout keeps `if: !cancelled()`
// (so a failing leg does not stop later legs from attempting to run), but
// only the Cygwin *setup* step (a flaky installer download, not a code
// failure) may carry continue-on-error. Every other step's failure now fails
// the job directly, with no hand-written logic in between to go stale.
func TestPRWindowsMakeShellLegsFailTheJob(t *testing.T) {
pr := readCIWorkflow(t, "pr.yml")
// Only the install/exercise "leg" steps need `if: !cancelled()` (so a
// failing leg does not stop a later, independent leg from attempting to
// run) and the no-continue-on-error rule; checkout/setup steps that every
// leg depends on are deliberately left to run unconditionally.
legSteps := map[string][]string{
"windows-make-shell": {
"Install native GNU Make", "Set up MSYS2 GNU Make", "Set up Cygwin GNU Make",
"Warn when Cygwin setup was skipped by a download failure",
"Exercise native Windows Make", "Exercise MSYS2-hosted Make", "Exercise Cygwin-hosted Make",
},
"test-windows-small": {
"Run doltversion package tests on Windows", "Run dbproxy server package tests on Windows",
},
}
for jobName, names := range legSteps {
job := pr.job(t, jobName)
for _, name := range names {
step := job.step(t, name)
if !strings.Contains(step.If, "!cancelled()") {
t.Errorf("%s step %q if = %q, want it to contain !cancelled() so later legs still run after an earlier one fails", jobName, step.Name, step.If)
}
allowCOE := jobName == "windows-make-shell" && step.ID == "cygwin"
if stepContinueOnError(step) && !allowCOE {
t.Errorf("%s step %q (id %q) has continue-on-error; only windows-make-shell's Cygwin setup step may, so every other leg's failure reds the job directly", jobName, step.Name, step.ID)
}
if allowCOE && !stepContinueOnError(step) {
t.Errorf("%s Cygwin setup step lost continue-on-error; a flaky installer download would now fail the job", jobName)
}
}
for _, step := range job.Steps {
if strings.HasPrefix(step.Name, "Fail if") {
t.Errorf("%s still has a hand-written aggregator step %q; SF-1 replaced aggregators with per-step failure propagation", jobName, step.Name)
}
if step.ID != "cygwin" && stepContinueOnError(step) {
t.Errorf("%s step %q (id %q) has continue-on-error; only windows-make-shell's Cygwin setup step may", jobName, step.Name, step.ID)
}
}
}
// The aggregator's job outputs (native/msys2/cygwin) are gone too
// (review N-3): nothing reads them once ci-gate only needs the job's own
// .result.
windowsMakeShell := pr.job(t, "windows-make-shell")
if len(windowsMakeShell.Outputs) != 0 {
t.Errorf("windows-make-shell outputs = %v, want none (N-3: they were unused)", windowsMakeShell.Outputs)
}
}
// TestPRWindowsMakeShellSetupStepsHaveTimeouts is review SF-2 (2026-10-03):
// the Cygwin and MSYS2 setup steps download third-party installers from the
// network; a step timeout bounds how long a hang can block the later legs'
// exercise steps behind it, instead of consuming the whole 30-minute job
// timeout (which would also lose native/msys2 coverage, since their exercise
// steps would never get a chance to run before the job-level cancel).
func TestPRWindowsMakeShellSetupStepsHaveTimeouts(t *testing.T) {
pr := readCIWorkflow(t, "pr.yml")
job := pr.job(t, "windows-make-shell")
for _, name := range []string{"Set up MSYS2 GNU Make", "Set up Cygwin GNU Make"} {
step := job.step(t, name)
if step.TimeoutMinutes <= 0 {
t.Errorf("windows-make-shell step %q has no timeout-minutes", name)
}
}
}
type bazelWorkflowCall struct {
Inputs map[string]struct {
Type string `yaml:"type"`
Default string `yaml:"default"`
} `yaml:"inputs"`
Secrets map[string]struct {
Required bool `yaml:"required"`
} `yaml:"secrets"`
Outputs map[string]struct {
Value string `yaml:"value"`
} `yaml:"outputs"`
}
func readBazelWorkflowCall(t *testing.T) bazelWorkflowCall {
t.Helper()
path := filepath.Join(sourceRepoRoot(t), ".github", "workflows", bazelWorkflowName)
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var doc struct {
On struct {
WorkflowCall bazelWorkflowCall `yaml:"workflow_call"`
} `yaml:"on"`
}
if err := yaml.Unmarshal(data, &doc); err != nil {
t.Fatalf("parse %s: %v", path, err)
}
return doc.On.WorkflowCall
}
// pr.yml calls bazel.yml once per PR and merge group, and its ci-gate
// requires the Bazel lanes, which are the only run of the Linux Go test tiers
// (ga-96smfk.22 retired the legacy jobs they mirrored in pr.yml and
// pr-risk.yml).
func TestBazelLaneIsGated(t *testing.T) {
workflow := readCIWorkflow(t, bazelWorkflowName)
call := readBazelWorkflowCall(t)
// Every job is the rbe job, a gated lane or an advisory lane (review D1
// F4): a new job cannot join the call without a decision about the gate.
wantCallOutputs := map[string]string{
"rbe-enabled": "${{ jobs." + bazelRBEJobName + ".outputs.enabled }}",
"rbe-mode": "${{ jobs." + bazelRBEJobName + ".outputs.mode }}",
}
for name, job := range workflow.Jobs {
if name == bazelRBEJobName || isBazelRRCJob(name) {
continue // the rrc jobs never run in a PR call (bazel_rrc_test.go)
}
if _, shadow := bazelShadowLanes[name]; shadow {
// A shadow lane reports into nothing: no job outputs, so no
// workflow_call output and no gate id
// (TestBazelDoltRaceShadowIsAdvisory).
if len(job.Outputs) != 0 {
t.Errorf("shadow lane %s outputs = %v, want none", name, job.Outputs)
}
continue
}
_, gated := bazelLaneGateIDs[name]
_, advisory := bazelAdvisoryLanes[name]
_, flagGated := bazelFlagGatedLanes[name]
if n := btoi(gated) + btoi(advisory) + btoi(flagGated); n != 1 {
t.Errorf("%s job %s: gated=%v advisory=%v flag-gated=%v; add it to exactly one of bazelLaneGateIDs (with a CI_GATE_REQUIRED id in pr.yml), bazelFlagGatedLanes (with its pr.yml flag) or bazelAdvisoryLanes (with the reason)",
bazelWorkflowName, name, gated, advisory, flagGated)
}
// Each lane reports its job.status from an always() last step.
if !reflect.DeepEqual(job.Outputs, map[string]string{"result": "${{ steps.result.outputs.result }}"}) {
t.Errorf("%s outputs = %v, want only result from the result step", name, job.Outputs)
}
last := job.Steps[len(job.Steps)-1]
if last.ID != "result" || last.If != "${{ always() }}" || !reflect.DeepEqual(last.Env, map[string]string{"JOB_STATUS": "${{ job.status }}"}) ||
last.Run != `echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT"` {
t.Errorf("%s last step = %+v; want the always() job.status recorder", name, last)
}
wantCallOutputs[name] = "${{ jobs." + name + ".outputs.result }}"
}
for name := range bazelLaneGateIDs {
if _, ok := workflow.Jobs[name]; !ok {
t.Errorf("bazelLaneGateIDs lists %s, which %s does not have", name, bazelWorkflowName)
}
}
for name := range bazelAdvisoryLanes {
if _, ok := workflow.Jobs[name]; !ok {
t.Errorf("bazelAdvisoryLanes lists %s, which %s does not have", name, bazelWorkflowName)
}
}
for name := range bazelFlagGatedLanes {
if _, ok := workflow.Jobs[name]; !ok {
t.Errorf("bazelFlagGatedLanes lists %s, which %s does not have", name, bazelWorkflowName)
}
}
gotCallOutputs := map[string]string{}
for name, out := range call.Outputs {
gotCallOutputs[name] = out.Value
}
if !reflect.DeepEqual(gotCallOutputs, wantCallOutputs) {
t.Errorf("workflow_call outputs = %v, want %v", gotCallOutputs, wantCallOutputs)
}
// One caller on PR events: pr.yml's bazel job, with the four RBE secrets
// only, bazelCallPermissions (contents: read, plus what the rrc jobs,
// which never run on a PR, ask for), and no if (the rbe job decides).
pr := readCIWorkflow(t, "pr.yml")
bazel := pr.job(t, "bazel")
if bazel.Uses != "./.github/workflows/"+bazelWorkflowName || bazel.If != "" || len(bazel.Needs) != 0 {
t.Errorf("pr.yml bazel job uses=%q if=%q needs=%v; want an unconditional call of %s", bazel.Uses, bazel.If, bazel.Needs, bazelWorkflowName)
}
if !reflect.DeepEqual(bazel.Permissions, bazelCallPermissions) {
t.Errorf("pr.yml bazel job permissions = %v, want %v", bazel.Permissions, bazelCallPermissions)
}
if !reflect.DeepEqual(bazel.With, bazelPRCallWith) {
t.Errorf("pr.yml bazel job with = %v, want exactly %v (no rbe or other override)", bazel.With, bazelPRCallWith)
}
if in := call.Inputs["integration"]; in.Type != "string" || in.Default != "on" {
t.Errorf("workflow_call input integration = %+v, want type string, default on (a boolean reads null as false on push)", in)
}
var declared []string
for name := range call.Secrets {
declared = append(declared, name)
if call.Secrets[name].Required {
t.Errorf("workflow_call secret %s is required; fork and Dependabot runs have none", name)
}
}
sort.Strings(declared)
var wantSecrets []string
for name := range bazelCallSecrets {
wantSecrets = append(wantSecrets, name)
}
sort.Strings(wantSecrets)
if !reflect.DeepEqual(declared, wantSecrets) {
t.Errorf("workflow_call secrets = %v, want %v", declared, wantSecrets)
}
callers := 0
entries, err := os.ReadDir(filepath.Join(sourceRepoRoot(t), ".github", "workflows"))
if err != nil {
t.Fatal(err)
}
for _, entry := range entries {
if !strings.HasSuffix(entry.Name(), ".yml") || entry.Name() == bazelWorkflowName {
continue
}
for jobName, job := range readCIWorkflow(t, entry.Name()).Jobs {
if !strings.HasSuffix(job.Uses, "/"+bazelWorkflowName) {
continue
}
callers++
// A caller forwards the RBE secrets: it must never run in a
// fork's privileged context (review D1 v2 N5). The one
// deliberate exception is bazel-farm.yml, whose pull_request_target
// run is limited to allowlisted fork authors and whose every
// security property TestBazelFarmWorkflowSecurity pins.
triggers := yamlMapKeys(readYAMLNode(t, filepath.Join(".github", "workflows", entry.Name())), "on")
for _, trigger := range triggers {
if trigger == "workflow_run" || (trigger == "pull_request_target" && entry.Name() != bazelFarmWorkflowName) {
t.Errorf("%s calls %s and has trigger %s; a caller may not run with secrets in a fork PR's context", entry.Name(), bazelWorkflowName, trigger)
}
}
if entry.Name() == bazelFarmWorkflowName && !reflect.DeepEqual(triggers, []string{"pull_request_target"}) {
t.Errorf("%s triggers = %v, want exactly [pull_request_target]", entry.Name(), triggers)
}
if entry.Name() != "pr.yml" && entry.Name() != "nightly.yml" && entry.Name() != bazelFarmWorkflowName {
t.Errorf("%s job %s calls %s; only pr.yml (PRs), nightly.yml and %s (trusted forks) may", entry.Name(), jobName, bazelWorkflowName, bazelFarmWorkflowName)
}
got := map[string]string{}
if m, ok := job.Secrets.(map[string]any); ok {
for k, v := range m {
got[k] = fmt.Sprint(v)
}
}
// bazel-farm.yml is the one deliberate exception (B1, security
// review of bdef342d5): its pull_request_target run must never
// carry the rbe-prewarm app secret, so it gets the four-secret
// map instead of the full five pr.yml and nightly.yml pass.
want := bazelCallSecrets
if entry.Name() == bazelFarmWorkflowName {
want = bazelFarmCallSecrets
}
if !reflect.DeepEqual(got, want) {
t.Errorf("%s job %s secrets = %v, want exactly %v (never inherit)", entry.Name(), jobName, job.Secrets, want)
}
}
}
if callers != 3 {
t.Errorf("%d jobs call %s, want 3 (pr.yml, nightly.yml, %s)", callers, bazelWorkflowName, bazelFarmWorkflowName)
}
// The gate: needs the call and requires exactly BAZEL plus one id per
// gated lane, each read from the lane's output (a missing output reads
// as skipped). BAZEL itself comes from bazel-gate.sh, never the env.
gate := pr.job(t, "ci-gate")
evaluate := gate.step(t, "Evaluate CI gate")
required := strings.Fields(evaluate.Env["CI_GATE_REQUIRED"])
if !contains(gate.Needs, "bazel") {
t.Errorf("ci-gate needs = %v, want bazel", gate.Needs)
}
// Plus the remote-execution check (TestPRGateRequiresRemoteBazelLanes).
wantBazelIDs := []string{bazelAggregateGateID, bazelRemoteGateID}
for lane, id := range bazelLaneGateIDs {
// F3: package-mcp/package-npm's ids (PACKAGE_MCP, PACKAGE_NPM) do not
// follow the "BAZEL" naming convention the prefix filter below
// groups by - they predate bazel.yml and keep their names - so they
// are checked here but excluded from wantBazelIDs.
if !bazelPackageJobs[lane] {
wantBazelIDs = append(wantBazelIDs, id)
}
if want := "${{ needs.bazel.outputs." + lane + " || 'skipped' }}"; evaluate.Env[id] != want {
t.Errorf("ci-gate env %s = %q, want %q", id, evaluate.Env[id], want)
}
}
sort.Strings(wantBazelIDs)
var gotBazelIDs []string
for _, id := range required {
if strings.HasPrefix(id, "BAZEL") {
gotBazelIDs = append(gotBazelIDs, id)
}
}
sort.Strings(gotBazelIDs)
if !reflect.DeepEqual(gotBazelIDs, wantBazelIDs) {
t.Errorf("ci-gate CI_GATE_REQUIRED Bazel ids = %v, want exactly %v", gotBazelIDs, wantBazelIDs)
}
if _, ok := evaluate.Env[bazelAggregateGateID]; ok {
t.Errorf("ci-gate env sets %s; it must come from %s aggregate", bazelAggregateGateID, bazelGateScript)
}
// Flag-gated lanes: read like a gated lane, never statically required,
// added by the run script exactly when the committed flag is "true",
// and the flag's value is pinned (flipping it is a reviewed one-line
// change, here and in pr.yml).
for lane, g := range bazelFlagGatedLanes {
if want := "${{ needs.bazel.outputs." + lane + " || 'skipped' }}"; evaluate.Env[g.id] != want {
t.Errorf("ci-gate env %s = %q, want %q", g.id, evaluate.Env[g.id], want)
}
if contains(required, g.id) {
t.Errorf("CI_GATE_REQUIRED statically lists %s; only the %s conditional in the run script may add it", g.id, g.flag)
}
if got := pr.Env[g.flag]; got != g.want {
t.Errorf("pr.yml top-level env %s = %q, want %q (update bazelFlagGatedLanes in the same change that flips it)", g.flag, got, g.want)
}
frag := bazelFlagGateRun(g)
if strings.Count(evaluate.Run, frag) != 1 {
t.Errorf("ci-gate run script lacks exactly one\n%s", frag)
} else if strings.Index(evaluate.Run, frag) > strings.Index(evaluate.Run, "export CI_GATE_REQUIRED") {
t.Errorf("ci-gate run script adds %s after export CI_GATE_REQUIRED", g.id)
}
}
for id, value := range evaluate.Env {
for lane := range bazelAdvisoryLanes {
if strings.Contains(value, "outputs."+lane) {
t.Errorf("ci-gate env %s reads advisory lane %s, which is not in the PR call (%s)", id, lane, bazelAdvisoryLanes[lane])
}
}
}
for key, want := range map[string]string{
"BAZEL_CALL": "${{ needs.bazel.result }}",
"BAZEL_RBE_MODE": "${{ needs.bazel.outputs.rbe-mode }}",
"BAZEL_RBE_ENABLED": "${{ needs.bazel.outputs.rbe-enabled }}",
} {
if evaluate.Env[key] != want {
t.Errorf("ci-gate env %s = %q, want %q", key, evaluate.Env[key], want)
}
}
// The gate reads the call's decision; it never re-derives it.
rederive := regexp.MustCompile(`(?i)RBE_WEST_WORKERS|head\.repo\.fork|HEAD_REPO_FORK|github\.actor|dependabot|secrets\.RBE`)
for key, value := range evaluate.Env {
if rederive.MatchString(key + "=" + value) {
t.Errorf("ci-gate env %s=%q re-derives the Bazel execution mode; read needs.bazel.outputs.rbe-mode", key, value)
}
}
if rederive.MatchString(evaluate.Run) {
t.Errorf("ci-gate run re-derives the Bazel execution mode:\n%s", evaluate.Run)
}
for _, line := range strings.Split(readPolicyFile(t, sourceRepoRoot(t), bazelGateScript), "\n") {
if !strings.HasPrefix(strings.TrimSpace(line), "#") && (rederive.MatchString(line) || strings.Contains(line, "GITHUB_EVENT")) {
t.Errorf("%s re-derives the execution mode: %q", bazelGateScript, line)
}
}
// The retired legacy jobs stay retired: nothing in pr.yml or
// pr-risk.yml runs the Go test tiers beside the Bazel lanes.
for _, job := range []string{"build-artifacts", "pr-core-wrapper", "check-cmd-bd-puregeo-tests", "bazel-coverage"} {
if _, ok := pr.Jobs[job]; ok || contains(gate.Needs, job) {
t.Errorf("pr.yml has retired job %s; the Bazel lanes run its tests", job)
}
}
risk := readCIWorkflow(t, "pr-risk.yml")
if got := sortedKeys(risk.Jobs); !reflect.DeepEqual(got, []string{"ci-gate", "test-nix"}) {
t.Errorf("pr-risk.yml jobs = %v, want only [ci-gate test-nix]; the Bazel lanes in pr.yml run the Go test tiers", got)
}
// PR Risk does not run the lane a second time.
for jobName, job := range risk.Jobs {
if strings.Contains(job.Uses, "bazel") || contains(job.Needs, "bazel") {
t.Errorf("pr-risk.yml job %s runs or needs the Bazel lane; pr.yml owns it", jobName)
}
}
}
// A called workflow's workflow-level concurrency group is evaluated in the
// caller's context, where github.workflow is the caller's name: a group equal
// to the caller's own deadlocks and GitHub cancels the call (BAZEL cancelled,
// every PR red). So bazel.yml's group never uses github.workflow and, for
// every event its callers run on, differs from each caller's group (review
// D1 v2 N4).
func TestBazelCallConcurrencyDiffersFromCallers(t *testing.T) {
group := func(file string) string {
t.Helper()
var doc struct {
Concurrency struct {
Group string `yaml:"group"`
} `yaml:"concurrency"`
}
if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+file)), &doc); err != nil {
t.Fatal(err)
}
return doc.Concurrency.Group
}
bazelGroup := group(bazelWorkflowName)
if bazelGroup == "" || strings.Contains(bazelGroup, "github.workflow") {
t.Fatalf("%s concurrency group = %q; want a fixed prefix, never github.workflow (the caller's name in a call)", bazelWorkflowName, bazelGroup)
}
exprRe := regexp.MustCompile(`\$\{\{\s*(.*?)\s*\}\}`)
eval := func(g, workflowName, event string) string {
t.Helper()
ctx := map[string]string{
"github.workflow": workflowName,
"github.event_name": event,
"github.ref": "refs/pull/123/merge",
"github.event.pull_request.number || github.ref": "123",
"github.event.pull_request.number": "123",
}
if event != "pull_request" && event != "pull_request_target" {
ctx["github.ref"] = "refs/heads/gh-readonly-queue/main/pr-123"
ctx["github.event.pull_request.number || github.ref"] = ctx["github.ref"]
}
return exprRe.ReplaceAllStringFunc(g, func(m string) string {
v, ok := ctx[exprRe.FindStringSubmatch(m)[1]]
if !ok {
t.Fatalf("concurrency group %q: cannot evaluate %s", g, m)
}
return v
})
}
for _, caller := range []string{"pr.yml", "nightly.yml", bazelFarmWorkflowName} {
callerGroup := group(caller)
if callerGroup == "" {
continue // no workflow-level group, nothing to collide with
}
name := yamlScalar(readYAMLNode(t, filepath.Join(".github", "workflows", caller)), "name")
for _, event := range []string{"pull_request", "pull_request_target", "merge_group", "push", "schedule", "workflow_dispatch"} {
if a, b := eval(bazelGroup, name, event), eval(callerGroup, name, event); a == b {
t.Errorf("%s event %s: %s's concurrency group %q equals the caller's; the call would deadlock", caller, event, bazelWorkflowName, a)
}
}
}
}
// bazelLaneRunModes: the rbe modes in which a lane's `if:` runs it, for a
// call with these inputs. Only the forms TestBazelWorkflowJobsAndExecutionMode
// allows are known. GitHub's != on strings is case-insensitive.
func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string) map[string]bool {
t.Helper()
switch ifExpr {
case "${{ needs.rbe.outputs.mode != 'skip' }}":
return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true, "cache": true, "local": true}
case "${{ needs.rbe.outputs.enabled == 'true' }}":
return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true}
case bazelIntegIf:
if strings.EqualFold(with["integration"], "off") {
return map[string]bool{}
}
return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true, "cache": true}
case bazelServerStorageIf:
// Remote-only (unlike bazelIntegIf, no mode cache run), but also
// honors `integration` like bazel-integration/bazel-cmd-dolt
// (gastownhall/beads#7486 review).
if strings.EqualFold(with["integration"], "off") {
return map[string]bool{}
}
return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true}
case bazelPackageGatesIf:
// F3: unlike every other lane, these two never skip because of the
// rbe job's decision - only because the caller turned them off
// entirely (pr.yml always passes "on").
if strings.EqualFold(with["package-gates"], "on") {
return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true, "cache": true, "local": true, "skip": true}
}
return map[string]bool{}
case bazelRBEPrewarmIf:
// Mode remote only (TestBazelWorkflowJobsAndExecutionMode's comment
// on the job: B1 removed fork-rw). Unlike the historical advisory
// lanes, this one legitimately runs in pr.yml's own call (see
// bazelAdvisoryLanes' name-specific exception in
// TestBazelGateSimulation below).
return map[string]bool{"remote": true}
case bazelDoltRaceJobIf:
// Mode remote, and only on pull_request and merge_group events:
// within pr.yml's call (the only PR-time caller these simulations
// model) that is mode remote. bazel-farm.yml's pull_request_target
// call, push, nightly and dispatch never run it
// (TestBazelDoltRaceShadowRunsOnlyInRemotePRCalls).
return map[string]bool{"remote": true}
case bazelRRCSeedJobIf, bazelRRCVerifyJobIf:
// The remote repo contents cache's writer (push to main) and
// nightly check (schedule, dispatch): never in a pull_request,
// merge_group or pull_request_target call, in any mode.
return map[string]bool{}
}
t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr)
return nil
}
// bazelGateScenario: what pr.yml's ci-gate sees of one bazel.yml call.
type bazelGateScenario struct {
name string
event string
mode, enabled string // the call's rbe-mode / rbe-enabled outputs
call string // needs.bazel.result
outputs map[string]string // the lanes' outputs ("" = not reported)
// Non-Bazel needs' results that differ from success.
results map[string]string
// pr.yml's top-level flags the run script reads (the workflow env is
// not part of the step's env): bazelFlagGatedLanes' flags.
flags map[string]string
wantPass bool
wantMention string // a red gate must name this id
}
// fastChecksLegacyJobNames: pr.yml's fast-checks job (F7a) output id -> the
// standalone job name it replaced. Lets the gate simulation resolve
// needs.fast-checks.outputs.* through the same prLegacyJobResult lookup
// (and sc.results overrides) the pre-fold standalone jobs used.
var fastChecksLegacyJobNames = map[string]string{
"build-tags": "check-build-tags",
"version-consistency": "check-version-consistency",
"migration-hygiene": "check-migration-hygiene",
"no-beads-changes": "check-no-beads-changes",
"fmt": "fmt-check",
}
// runPRGateStep runs pr.yml's actual "Evaluate CI gate" step (its run block,
// under GitHub's bash flags) with its env evaluated for the scenario: every
// non-Bazel need succeeded; Bazel expressions read the scenario. An env
// expression of any other form fails the test, so the simulation cannot
// silently drift from the workflow.
func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (bool, string) {
t.Helper()
expr := regexp.MustCompile(`^\$\{\{ needs\.([A-Za-z0-9_-]+)\.(result|outputs\.([A-Za-z0-9_-]+))( \|\| 'skipped')? \}\}$`)
env := []string{"PATH=" + os.Getenv("PATH"), "GITHUB_EVENT_NAME=" + sc.event}
for key, value := range sc.flags {
env = append(env, key+"="+value)
}
for key, value := range step.Env {
if !strings.Contains(value, "${{") {
env = append(env, key+"="+value)
continue
}
m := expr.FindStringSubmatch(value)
if m == nil {
t.Fatalf("ci-gate env %s = %q: the gate simulation cannot evaluate it", key, value)
}
var got string
switch {
case m[1] == "fast-checks" && m[2] != "result":
// F7a: fast-checks folds five formerly-standalone jobs into one
// job's steps; ci-gate reads each step's outcome via the job's
// outputs instead of a separate job's .result. Resolve through
// the same prLegacyJobResult the standalone jobs used, keyed by
// their old job name, so sc.results overrides (and the default
// "success") work exactly as they did before the fold.
legacy, ok := fastChecksLegacyJobNames[m[3]]
if !ok {
t.Fatalf("ci-gate env %s = %q: fast-checks has no such step output", key, value)
}
got = prLegacyJobResult(legacy, sc)
case m[1] != "bazel" && m[2] == "result":
got = prLegacyJobResult(m[1], sc)
case m[1] != "bazel":
t.Fatalf("ci-gate env %s = %q: the gate simulation cannot evaluate it", key, value)
case m[2] == "result":
got = sc.call
case m[3] == "rbe-mode":
got = sc.mode
case m[3] == "rbe-enabled":
got = sc.enabled
default:
got = sc.outputs[m[3]]
}
if got == "" && m[4] != "" {
got = "skipped"
}
env = append(env, key+"="+got)
}
cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step.Run)
cmd.Dir = sourceRepoRoot(t)
cmd.Env = env
out, err := cmd.CombinedOutput()
return err == nil, string(out)
}
// The gate over every execution mode (review D1 F3): the skip script allows
// exactly the skips the lanes' own `if:`s produce in that mode, and for every
// lane that should run, that lane alone skipped, failed or cancelled turns
// pr.yml's actual gate step red, whatever the aggregate says; a missing or
// inconsistent mode, or an aggregate failure no lane explains, turns it red;
// and a mode that did not execute the lanes on rbe-west (skip, local, cache)
// is red even when everything that ran passed (BAZEL_REMOTE).
func TestBazelGateSimulation(t *testing.T) {
requireHostTool(t, "bash")
root := sourceRepoRoot(t)
workflow := readCIWorkflow(t, bazelWorkflowName)
pr := readCIWorkflow(t, "pr.yml")
step := pr.job(t, "ci-gate").step(t, "Evaluate CI gate")
callWith := pr.job(t, "bazel").With
lanes := map[string]map[string]bool{} // lane -> modes it runs in, in pr.yml's call
for name, job := range workflow.Jobs {
if name == bazelRBEJobName {
continue
}
lanes[name] = bazelLaneRunModes(t, name, job.If, callWith)
// Advisory lanes are normally not in the PR call at all; the one
// exception is rbe-prewarm, which is advisory (never gated, never a
// `needs`: TestBazelLaneIsGatedAlongsideLegacy) but, unlike every
// other advisory lane this map has ever held, is deliberately part
// of every pr.yml call so it can dispatch before the lanes below
// need capacity - see bazel.yml's own comment on the job.
if _, advisory := bazelAdvisoryLanes[name]; advisory && name != bazelRBEPrewarmJobName && len(lanes[name]) != 0 {
t.Errorf("advisory lane %s runs in pr.yml's call in modes %v; turn it off there (%s)", name, lanes[name], bazelAdvisoryLanes[name])
}
if _, gated := bazelLaneGateIDs[name]; gated && !lanes[name]["remote"] {
t.Errorf("gated lane %s does not run in pr.yml's call even in remote mode", name)
}
}
enabledFor := bazelModeEnabled
runScript := func(mode, enabled, arg string) string {
t.Helper()
cmd := exec.Command("bash", bazelGateScript, arg)
cmd.Dir = root
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=" + mode, "BAZEL_RBE_ENABLED=" + enabled}
out, err := cmd.Output()
if err != nil {
t.Fatalf("%s %s (mode %q, enabled %q): %v", bazelGateScript, arg, mode, enabled, err)
}
return strings.TrimSpace(string(out))
}
var scenarios []bazelGateScenario
for _, mode := range bazelRBEModes {
// Expected skips, from the jobs' ifs alone.
var wantSkips []string
anyRuns := false
for lane, modes := range lanes {
// F3: bazel-gate.sh's skip list and its BAZEL aggregate are
// mode-derived only; it has never heard of package-mcp/npm
// (their own PACKAGE_MCP/PACKAGE_NPM scenarios are exercised
// below, by lane, not through this script).
if bazelPackageJobs[lane] {
continue
}
if modes[mode] {
anyRuns = true
} else if id, gated := bazelLaneGateIDs[lane]; gated {
wantSkips = append(wantSkips, id)
} else if g, flagGated := bazelFlagGatedLanes[lane]; flagGated {
// Listed even while advisory, so flipping the flag is the
// only change that makes it required.
wantSkips = append(wantSkips, g.id)
}
}
if !anyRuns {
wantSkips = append(wantSkips, bazelAggregateGateID)
}
sort.Strings(wantSkips)
gotSkips := strings.Fields(runScript(mode, enabledFor(mode), "skips"))
sort.Strings(gotSkips)
if !reflect.DeepEqual(gotSkips, wantSkips) && !(len(gotSkips) == 0 && len(wantSkips) == 0) {
t.Errorf("mode %s: %s skips = %v, want exactly %v (the ids whose job if is false)", mode, bazelGateScript, gotSkips, wantSkips)
}
base := func() map[string]string {
out := map[string]string{}
for lane, modes := range lanes {
if modes[mode] {
out[lane] = "success"
}
}
return out
}
with := func(kv ...string) map[string]string {
out := base()
for i := 0; i+1 < len(kv); i += 2 {
out[kv[i]] = kv[i+1]
}
return out
}
// Flags at their committed values, unless a scenario overrides one.
committedFlags := map[string]string{}
for _, g := range bazelFlagGatedLanes {
committedFlags[g.flag] = pr.Env[g.flag]
}
scFlags := func(name, call string, outputs map[string]string, flags map[string]string, pass bool, mention string) {
// The lanes are the Linux Go test tiers' only run: a mode that
// executed nothing on rbe-west is red whatever else holds
// (BAZEL_REMOTE, TestPRGateRequiresRemoteBazelLanes).
if !bazelRemoteModes[mode] {
if pass || mention == "" {
mention = bazelRemoteGateID
}
pass = false
}
all := copyMap(committedFlags)
for k, v := range flags {
all[k] = v
}
for _, event := range []string{"pull_request", "merge_group"} {
scenarios = append(scenarios, bazelGateScenario{
name: mode + "/" + event + "/" + name, event: event, mode: mode, enabled: enabledFor(mode),
call: call, outputs: outputs, flags: all, wantPass: pass, wantMention: mention,
})
}
}
sc := func(name, call string, outputs map[string]string, pass bool, mention string) {
scFlags(name, call, outputs, nil, pass, mention)
}
sc("every lane as designed", "success", base(), true, "")
sc("aggregate failure no lane explains", "failure", base(), false, bazelAggregateGateID)
sc("aggregate cancelled", "cancelled", base(), false, bazelAggregateGateID)
sc("aggregate skipped", "skipped", base(), mode == "skip", bazelAggregateGateID)
for lane, modes := range lanes {
if g, flagGated := bazelFlagGatedLanes[lane]; flagGated {
off := map[string]string{g.flag: "false"}
on := map[string]string{g.flag: "true"}
// Advisory (flag "false"): continue-on-error keeps the
// aggregate green, and nothing the lane reports reddens
// the gate.
for _, result := range []string{"", "failure", "cancelled"} {
scFlags(lane+" reports "+result+" while advisory", "success", with(lane, result), off, true, "")
}
// Required (flag "true"): exactly a gated lane.
scFlags(lane+" required, as designed", "success", base(), on, true, "")
if !modes[mode] {
scFlags(lane+" required, ran and failed", "success", with(lane, "failure"), on, false, g.id)
continue
}
for _, result := range []string{"", "failure", "cancelled"} {
scFlags(lane+" required, alone "+result, "success", with(lane, result), on, false, g.id)
}
continue
}
id, gated := bazelLaneGateIDs[lane]
if !gated {
// Advisory: not in the call, so it cannot report, and
// nothing about it excuses the aggregate.
for _, result := range []string{"failure", "cancelled"} {
sc(lane+" reports "+result+" though off", "failure", with(lane, result), false, bazelAggregateGateID)
}
continue
}
if !modes[mode] {
// Skipped by design (baseline); if it ran anyway and failed,
// the gate still sees it.
sc(lane+" ran and failed", "failure", with(lane, "failure"), false, id)
continue
}
for _, call := range []string{"success", "failure"} {
sc(lane+" alone skipped, aggregate "+call, call, with(lane, ""), false, id)
sc(lane+" failed, aggregate "+call, call, with(lane, "failure"), false, id)
sc(lane+" cancelled, aggregate "+call, call, with(lane, "cancelled"), false, id)
}
sc(lane+" cancelled, aggregate cancelled", "cancelled", with(lane, "cancelled"), false, id)
}
}
// A missing or inconsistent decision (the rbe job failed, the call never
// started, or the outputs disagree) allows no skip and fails the gate.
for _, bad := range []struct{ mode, enabled string }{
{"", ""}, {"remote", "false"}, {"local", "true"}, {"cache", "true"}, {"skip", "true"}, {"remote", ""}, {"bogus", "false"}, {"REMOTE", "true"}, {"CACHE", "false"},
{"fork-ro", "false"}, {"fork-rw", "false"}, {"fork-ro", ""}, {"FORK-RO", "true"}, {"fork", "true"}, {"fork-", "true"}, {"fork-rx", "true"},
} {
if got := runScript(bad.mode, bad.enabled, "skips"); got != "" {
t.Errorf("mode %q enabled %q: skips = %q, want none", bad.mode, bad.enabled, got)
}
all := map[string]string{}
for lane := range lanes {
all[lane] = "success"
}
scenarios = append(scenarios,
bazelGateScenario{name: "invalid " + bad.mode + "/" + bad.enabled + ", every lane success", event: "pull_request",
mode: bad.mode, enabled: bad.enabled, call: "success", outputs: all, wantMention: bazelAggregateGateID},
bazelGateScenario{name: "invalid " + bad.mode + "/" + bad.enabled + ", nothing ran", event: "pull_request",
mode: bad.mode, enabled: bad.enabled, call: "failure", outputs: map[string]string{}, wantMention: bazelAggregateGateID},
)
}
for _, sc := range scenarios {
t.Run(sc.name, func(t *testing.T) {
pass, out := runPRGateStep(t, step, sc)
if pass != sc.wantPass {
t.Errorf("gate pass = %v, want %v (outputs %v, call %s)\n%s", pass, sc.wantPass, sc.outputs, sc.call, out)
}
if !sc.wantPass && sc.wantMention != "" && !regexp.MustCompile(`::error::`+sc.wantMention+`\b`).MatchString(out) {
t.Errorf("red gate does not name %s:\n%s", sc.wantMention, out)
}
})
}
}
// Artifact names are unique within a run, and a called workflow's uploads
// belong to the caller's run: a duplicate name makes the second upload fail
// (409) or a download pick the wrong file. So every run that calls bazel.yml
// (with its own inputs) must not upload any name twice (review D1 F7).
func TestBazelArtifactNamesUniqueInCallerRuns(t *testing.T) {
for _, caller := range []string{"pr.yml", "nightly.yml", bazelFarmWorkflowName} {
t.Run(caller, func(t *testing.T) {
uses := collectRunArtifactUploads(t, caller, nil, "")
var sawBazel bool
for _, u := range uses {
sawBazel = sawBazel || strings.Contains(u.where, "-> "+bazelWorkflowName+" job ")
}
if !sawBazel {
t.Fatalf("%s run has no upload from %s; the collector did not follow the call", caller, bazelWorkflowName)
}
for i, a := range uses {
if a.matrix && a.pattern == nil {
t.Errorf("%s: %s uploads fixed name %q from a matrix job; every leg collides", caller, a.where, a.name)
}
for _, b := range uses[i+1:] {
if bazelArtifactNamesCollide(a, b) {
t.Errorf("%s run: %s and %s both upload %q / %q", caller, a.where, b.where, a.name, b.name)
}
}
}
})
}
}
type runArtifactUpload struct {
name, where string
pattern *regexp.Regexp // nil when the name resolved to a literal
matrix bool
}
func bazelArtifactNamesCollide(a, b runArtifactUpload) bool {
switch {
case a.pattern == nil && b.pattern == nil:
return a.name == b.name
case a.pattern == nil:
return b.pattern.MatchString(a.name)
case b.pattern == nil:
return a.pattern.MatchString(b.name)
}
return a.name == b.name
}
// collectRunArtifactUploads lists the upload-artifact names of one run of
// file: its jobs' steps and, recursively, those of the local workflows its
// jobs call with their `with:` inputs (workflow_call defaults otherwise).
// inputs.X, `inputs.X || 'lit'` and env.X (job, then workflow env) resolve;
// any other expression (a matrix value) becomes a wildcard.
func collectRunArtifactUploads(t *testing.T, file string, with map[string]string, prefix string) []runArtifactUpload {
t.Helper()
path := filepath.Join(sourceRepoRoot(t), ".github", "workflows", file)
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var doc struct {
On struct {
WorkflowCall struct {
Inputs map[string]struct {
Default any `yaml:"default"`
} `yaml:"inputs"`
} `yaml:"workflow_call"`
} `yaml:"on"`
Env map[string]string `yaml:"env"`
Jobs map[string]struct {
Uses string `yaml:"uses"`
With map[string]any `yaml:"with"`
Env map[string]string `yaml:"env"`
Strategy struct {
Matrix any `yaml:"matrix"`
} `yaml:"strategy"`
Steps []ciWorkflowStep `yaml:"steps"`
} `yaml:"jobs"`
}
if err := yaml.Unmarshal(data, &doc); err != nil {
t.Fatalf("parse %s: %v", path, err)
}
inputs := map[string]string{}
for name, in := range doc.On.WorkflowCall.Inputs {
if in.Default != nil {
inputs[name] = fmt.Sprint(in.Default)
}
}
for k, v := range with {
inputs[k] = v
}
exprRe := regexp.MustCompile(`\$\{\{\s*(.*?)\s*\}\}`)
inputOr := regexp.MustCompile(`^inputs\.([A-Za-z0-9_-]+)(?:\s*\|\|\s*'([^']*)')?$`)
envRef := regexp.MustCompile(`^env\.([A-Za-z0-9_]+)$`)
var out []runArtifactUpload
var names []string
for name := range doc.Jobs {
names = append(names, name)
}
sort.Strings(names)
for _, jobName := range names {
job := doc.Jobs[jobName]
if strings.HasPrefix(job.Uses, "./.github/workflows/") {
callWith := map[string]string{}
for k, v := range job.With {
callWith[k] = fmt.Sprint(v)
}
called := strings.TrimPrefix(job.Uses, "./.github/workflows/")
out = append(out, collectRunArtifactUploads(t, called, callWith, prefix+file+" job "+jobName+" -> ")...)
continue
}
var resolve func(s string, depth int) (string, bool)
resolve = func(s string, depth int) (string, bool) {
literal := true
res := exprRe.ReplaceAllStringFunc(s, func(m string) string {
inner := exprRe.FindStringSubmatch(m)[1]
if in := inputOr.FindStringSubmatch(inner); in != nil {
if v := inputs[in[1]]; v != "" {
return v
}
return in[2]
}
if e := envRef.FindStringSubmatch(inner); e != nil && depth < 4 {
v, ok := job.Env[e[1]]
if !ok {
v, ok = doc.Env[e[1]]
}
if ok {
r, lit := resolve(v, depth+1)
literal = literal && lit
return r
}
}
literal = false
return "\x00"
})
return res, literal
}
for _, step := range job.Steps {
if actionFamily(step.Uses) != "actions/upload-artifact" {
continue
}
name, literal := resolve(step.With["name"], 0)
u := runArtifactUpload{name: name, where: prefix + file + " job " + jobName, matrix: job.Strategy.Matrix != nil}
if !literal {
parts := strings.Split(name, "\x00")
for i := range parts {
parts[i] = regexp.QuoteMeta(parts[i])
}
u.pattern = regexp.MustCompile("^" + strings.Join(parts, ".+") + "$")
u.name = strings.ReplaceAll(name, "\x00", "*")
}
out = append(out, u)
}
}
return out
}
// Every action in the Bazel lane is pinned to a full commit SHA, with the same
// released SHAs the Go lanes use, and the monolithic actions/cache is banned
// here too (restore everywhere, save only where the topology says).
func TestBazelWorkflowActionsArePinned(t *testing.T) {
steps := map[string][]ciWorkflowStep{
setupBazelActionDir + "/action.yml": readSetupBazelAction(t).Runs.Steps,
}
for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs {
steps[bazelWorkflowName+" "+name] = job.Steps
}
want := map[string]string{
setupNodeActionFamily: setupNodeSHA,
setupPythonActionFamily: setupPythonSHA,
"actions/checkout": checkoutSHA,
setupGoActionFamily: setupGoSHA,
cacheRestoreActionFamily: cacheSHA,
cacheSaveActionFamily: cacheSHA,
"actions/upload-artifact": uploadArtifactSHA,
"actions/download-artifact": downloadArtifactSHA,
// rbe-prewarm's mint step: same SHA/comment already used in this repo
// for the same action, in update-flake-lock.yml.
"actions/create-github-app-token": "bcd2ba49218906704ab6c1aa796996da409d3eb1",
}
for file, list := range steps {
for _, step := range list {
if step.Uses == "" || strings.HasPrefix(step.Uses, "./") {
continue
}
family, sha, found := strings.Cut(step.Uses, "@")
if !found || !actionPin.MatchString(sha) {
t.Errorf("%s step %q action %q is not pinned to a 40-hex commit SHA", file, step.Name, step.Uses)
continue
}
if isForbiddenGoCacheActionFamily(family) {
t.Errorf("%s step %q uses forbidden monolithic %q; use actions/cache/restore and /save", file, step.Name, family)
continue
}
wantSHA, known := want[family]
if !known {
t.Errorf("%s step %q uses %q, which this policy does not know; add its released SHA here", file, step.Name, family)
} else if sha != wantSHA {
t.Errorf("%s step %q action %q has SHA %q, want %q", file, step.Name, family, sha, wantSHA)
}
if family == setupGoActionFamily && step.With["cache"] != "false" {
t.Errorf("%s setup-go cache = %q, want false", file, step.With["cache"])
}
if family == cacheSaveActionFamily && file != bazelWorkflowName+" "+bazelJobName {
t.Errorf("%s step %q saves a cache; only %s's push-to-main step may", file, step.Name, bazelJobName)
}
}
for _, step := range list {
if strings.Contains(step.Run, "apt-get") {
t.Errorf("%s step %q runs apt-get; bound it (see ci_workflow_apt_get_timeout_test.go) or drop it", file, step.Name)
}
}
}
}
// The runner cache holds only content fixed by .bazelversion and
// MODULE.bazel.lock (repository cache, Bazelisk), so it is keyed on exactly
// that and written only by a push to main that missed the key; it never holds
// the generated rc or credentials.
func TestBazelWorkflowCacheTopology(t *testing.T) {
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName)
var saves []ciWorkflowStep
for _, step := range job.Steps {
if actionFamily(step.Uses) == cacheSaveActionFamily {
saves = append(saves, step)
}
}
if len(saves) != 1 {
t.Fatalf("%s has %d cache save steps, want 1", bazelWorkflowName, len(saves))
}
save := saves[0]
if save.If != bazelCacheSaveIf {
t.Errorf("cache save if = %q, want exactly %q", save.If, bazelCacheSaveIf)
}
if save.With["key"] != bazelCacheKey || save.With["path"] != bazelCachePath {
t.Errorf("cache save key/path = %q / %q, want %q / %q", save.With["key"], save.With["path"], bazelCacheKey, bazelCachePath)
}
var restore, writer ciWorkflowStep
for _, step := range readSetupBazelAction(t).Runs.Steps {
switch {
case actionFamily(step.Uses) == cacheRestoreActionFamily:
restore = step
case step.Env["BAZEL_CI_SECRET_DIR"] != "":
writer = step
}
}
if restore.With["path"] != bazelCachePath || restore.With["key"] != bazelCacheKey ||
strings.TrimSpace(restore.With["restore-keys"]) != bazelCacheKeyPrefix {
t.Errorf("setup-bazel restore path/key/restore-keys = %q / %q / %q; want the save step's path and key with a %q prefix fallback",
restore.With["path"], restore.With["key"], restore.With["restore-keys"], bazelCacheKeyPrefix)
}
if writer.Env["BAZEL_CI_CACHE_DIR"] != bazelCachePath {
t.Errorf("rc writer BAZEL_CI_CACHE_DIR = %q, want %q", writer.Env["BAZEL_CI_CACHE_DIR"], bazelCachePath)
}
if secret := writer.Env["BAZEL_CI_SECRET_DIR"]; !strings.HasPrefix(secret, "${{ runner.temp }}/") || strings.HasPrefix(secret, bazelCachePath) {
t.Errorf("rc writer BAZEL_CI_SECRET_DIR = %q; want a runner.temp directory outside the cached %q", secret, bazelCachePath)
}
}
// The lane runs the committed ci config over //..., reports the critical path
// and the go test equivalence even when tests fail, catches BUILD drift, and
// hands secrets to nothing but setup-bazel.
func TestBazelWorkflowRunsCIConfigWithReports(t *testing.T) {
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName)
if job.TimeoutMinutes == 0 {
t.Errorf("%s has no timeout-minutes", bazelJobName)
}
testStep := job.step(t, "bazel test //... --config=ci")
for _, required := range []string{"bazel test //... --config=ci", "--profile=", "--build_event_json_file=", "set -o pipefail"} {
if !strings.Contains(testStep.Run, required) {
t.Errorf("test step does not contain %q:\n%s", required, testStep.Run)
}
}
if strings.Contains(testStep.Run, "--config=remote-exec") {
t.Errorf("test step selects remote-exec itself; setup-bazel's rc does that only when secrets are present")
}
if i, j := strings.Index(testStep.Run, "bazel test"), strings.Index(testStep.Run, "--profile="); j < i {
t.Errorf("--profile is a command option and must follow `bazel test`")
}
for name, script := range map[string]string{
"Critical-path report": "tools/bazel/critpath.py",
"Go test equivalence": "tools/bazel/equivalence.py",
} {
step := job.step(t, name)
if !strings.Contains(step.Run, script) || !strings.Contains(step.If, "always()") {
t.Errorf("step %q must run %s under always(); run=%q if=%q", name, script, step.Run, step.If)
}
assertReportStepKeepsExitStatus(t, step, "python3 "+script)
}
assertTestStepKeepsExitStatus(t, testStep)
sync := job.step(t, "BUILD files in sync (gazelle, go_srcs, MODULE.bazel)")
for _, required := range []string{"make bazel-sync-check", "make bazel-sync\n", "git status --porcelain --untracked-files=all", "exit 1"} {
if !strings.Contains(sync.Run, required) {
t.Errorf("sync step does not contain %q:\n%s", required, sync.Run)
}
}
}
// bazel.yml publishes a Bazel-built bd under pr.yml's build-artifacts contract
// (artifact name, file names, checksum file, retention), so the jobs that
// download ci-build-artifacts can switch to it without other edits.
func TestBazelWorkflowPublishesBuildArtifacts(t *testing.T) {
// The layout and upload settings of the retired build-artifacts job
// (ga-96smfk.22), which bazel-test's artifact replaced.
prUpload := ciWorkflowStep{With: map[string]string{
"name": "ci-build-artifacts",
"retention-days": "1",
"if-no-files-found": "error",
}}
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName)
pkg := job.step(t, "Package bd (ci-build-artifacts layout)")
upload := job.step(t, "Upload build artifacts")
// Named by the build-artifact-name input, ci-build-artifacts by default;
// the run inside pr.yml renames it (TestBazelLaneIsGated).
if upload.With["name"] != "${{ inputs.build-artifact-name || '"+prUpload.With["name"]+"' }}" ||
readBazelWorkflowCall(t).Inputs["build-artifact-name"].Default != prUpload.With["name"] {
t.Errorf("artifact name = %q, want the build-artifact-name input defaulting to %q", upload.With["name"], prUpload.With["name"])
}
for _, key := range []string{"retention-days", "if-no-files-found"} {
if upload.With[key] != prUpload.With[key] {
t.Errorf("upload %s = %q, want %q", key, upload.With[key], prUpload.With[key])
}
}
if pkg.ID == "" || upload.If != "${{ always() && steps."+pkg.ID+".outcome == 'success' }}" {
t.Errorf("upload if = %q; want it gated on the package step's success", upload.If)
}
// The commands live in a script that sources .buildflags itself, so the
// build-tag scan (scripts/check-build-tags.sh) still covers bazel.yml.
const script = "scripts/ci/package-bazel-bd.sh"
if pkg.Run != "./"+script+` "$RUNNER_TEMP/bd-artifacts"` {
t.Errorf("package step = %q, want %s", pkg.Run, script)
}
if strings.Contains(readPolicyFile(t, bazelPolicyRoot(t), ".github/workflows/"+bazelWorkflowName), ".buildflags") {
t.Errorf("%s mentions .buildflags; sourcing it would exempt the whole file from the build-tag scan", bazelWorkflowName)
}
body := readPolicyFile(t, bazelPolicyRoot(t), script)
for _, required := range []string{
"set -euo pipefail", "source ./.buildflags",
"/bin/cmd/bd/bd_for_tests/bd", "bd-linux-gms-pure", "sha256sum bd-linux-gms-pure > SHA256SUMS",
"build-manifest.txt", "commit=", "go_version=", "build_tags=", "artifact=bd-linux-gms-pure",
} {
if !strings.Contains(body, required) {
t.Errorf("%s does not contain %q", script, required)
}
}
// test:ci must download the binary under --remote_download_minimal.
if !strings.Contains(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "test:ci --remote_download_regex=.*/bin/cmd/bd/bd_for_tests/bd$") {
t.Error(".bazelrc test:ci does not download //cmd/bd:bd_for_tests")
}
}
// bazel-doltserver is the only run of what pr.yml's container-backed jobs
// ("Test (storage domain + uow)", "Contract corpus") ran: --config=doltserver
// (hermetic dolt sql-servers, remote-executable), with a dolt-server target
// in every package those jobs ran. Nothing runs those suites against
// testcontainers any more: no docker lane, no requires-docker target, no
// image pull in bazel.yml, and neither job in pr.yml.
func TestBazelDoltJobMirrorsContainerJobs(t *testing.T) {
pr := readCIWorkflow(t, "pr.yml")
for _, name := range []string{"test-domain-uow", "contract-corpus"} {
if _, ok := pr.Jobs[name]; ok {
t.Errorf("pr.yml has job %s again; //...:*_dolt_test on the dolt-server lane runs its tests", name)
}
}
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDoltJobName)
for _, st := range job.Steps {
if strings.Contains(st.Run, "pull-dolt-image") || strings.Contains(st.Run, "docker") {
t.Errorf("%s step %q uses docker; the dolt-server lane needs none", bazelDoltJobName, st.Name)
}
}
if _, ok := job.Env["BAZEL_DOLT_LANE"]; ok {
t.Errorf("%s still selects its lane from BAZEL_DOLT_LANE; it runs --config=doltserver only", bazelDoltJobName)
}
test := job.step(t, "bazel test //... --config=doltserver")
if !strings.Contains(test.Run, "bazel test //... --config=doltserver ") || !strings.Contains(test.Run, "set -o pipefail") {
t.Errorf("dolt lane step does not run the lane over //...:\n%s", test.Run)
}
if strings.Contains(test.Run, "--config=remote-exec") {
t.Errorf("dolt lane step selects remote-exec itself; setup-bazel's rc does that only when secrets are present")
}
assertTestStepKeepsExitStatus(t, test)
rc := readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc")
if !strings.Contains(rc, "test:doltserver --test_tag_filters=dolt-server\n") {
t.Errorf(".bazelrc lacks %q", "test:doltserver --test_tag_filters=dolt-server")
}
if regexp.MustCompile(`(?m)^(build|test):docker\b`).MatchString(rc) {
t.Error(".bazelrc defines a docker config again; the dolt-server lane is hermetic")
}
// The packages those jobs ran (test-domain-uow: domain/..., uow,
// tracker/... and doctor/fix, and the server fingerprint, whose local
// half testutil's target runs; contract-corpus: protocol) each need a
// dolt-server target. Every such target, checked on its own, picks the
// local backend and fails closed, so a broken backend fails rather than
// skipping into a cached pass; the ones whose TestMain owns a server also
// set the package's own REQUIRE switch.
root := sourceRepoRoot(t)
for _, pkg := range []string{
"internal/storage/domain",
"internal/storage/domain/db",
"internal/storage/domain/fs",
"internal/storage/domain/git",
"internal/storage/uow",
"internal/testutil",
"internal/tracker",
"internal/tracker/conformance",
"cmd/bd/doctor/fix",
"cmd/bd/protocol",
} {
for _, err := range checkDoltServerRules(pkg, readPolicyFile(t, root, pkg+"/BUILD.bazel")) {
t.Error(err)
}
}
}
// doltServerRuleEnv is the rule env every dolt-server target must set, and
// doltServerExtraEnv what particular targets need on top: the TestMain
// switches that turn a server which cannot start into a failure.
var (
doltServerRuleEnv = []string{`"BEADS_TEST_DOLT_SERVER": "local"`, `"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"`}
doltServerExtraEnv = map[string][]string{
"fix_dolt_test": {`"BEADS_FIX_REQUIRE_DOLT": "1"`},
"protocol_dolt_test": {`"BEADS_PROTOCOL_REQUIRE_DOLT": "1"`},
// Without it every proxied test self-skips (the jobs' step env).
"bd_proxied_test": {`"BEADS_TEST_PROXIED_SERVER": "1"`},
// federation_test.go fails instead of skipping without a server.
"dolt_server_full_test": {`"BEADS_TEST_ENV_RUN_DOLT": "1"`},
}
// doltServerLaneTags are the tags of the lanes whose targets start
// hermetic dolt sql-servers; checkDoltServerRules holds each of them to
// the same rules.
doltServerLaneTags = []string{"dolt-server", "dolt-server-proxied", "dolt-server-integration", "dolt-server-cmd"}
bazelRuleNameRe = regexp.MustCompile(`(?m)^\s*name\s*=\s*"([^"]+)"`)
)
// bazelTopRules splits a BUILD file into its top-level calls: each starts at
// a line matching bazelTopRuleRe and ends at the next line that is just ")".
func bazelTopRules(build string) []string {
var rules []string
var cur []string
in := false
for _, line := range strings.Split(build, "\n") {
if !in && bazelTopRuleRe.MatchString(line) {
in, cur = true, nil
}
if in {
cur = append(cur, line)
if line == ")" || (len(cur) == 1 && strings.HasSuffix(strings.TrimSpace(line), ")")) {
rules = append(rules, strings.Join(cur, "\n"))
in = false
}
}
}
return rules
}
// doltServerRuleTag returns the dolt-server lane tag a rule carries, or "".
func doltServerRuleTag(rule string) string {
for _, tag := range doltServerLaneTags {
if strings.Contains(rule, `"`+tag+`"`) {
return tag
}
}
return ""
}
// checkDoltServerRules checks each rule tagged for a dolt-server lane
// (doltServerLaneTags) in one package's BUILD file on its own (a file-wide
// search would accept another target's env), and that the package has at
// least one.
func checkDoltServerRules(pkg, build string) []error {
var errs []error
found := 0
for _, rule := range bazelTopRules(stripStarlarkComments(build)) {
tag := doltServerRuleTag(rule)
if tag == "" {
continue
}
found++
name := "?"
if m := bazelRuleNameRe.FindStringSubmatch(rule); m != nil {
name = m[1]
}
where := "//" + pkg + ":" + name + " (" + tag + ")"
for _, env := range append(append([]string{}, doltServerRuleEnv...), doltServerExtraEnv[name]...) {
if !strings.Contains(rule, env) {
errs = append(errs, errors.New(where+" lacks rule env "+env))
}
}
if strings.Contains(rule, `"no-remote-exec"`) {
errs = append(errs, errors.New(where+" is tagged no-remote-exec; the lane runs on remote workers"))
}
}
if found == 0 {
errs = append(errs, errors.New(pkg+"/BUILD.bazel has no dolt-server target for the dolt-server lane"))
}
return errs
}
// A dolt-server rule is checked on its own: another target in the same file
// (here an untagged variant) carrying the env must not cover for it.
func TestCheckDoltServerRulesPerTarget(t *testing.T) {
const docker = `sh_test(
name = "uow_container_test",
env = {
"BEADS_TEST_DOLT_SERVER": "container",
"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1",
},
)
`
good := `load("@rules_shell//shell:sh_test.bzl", "sh_test")
` + docker + `
# Tags:
# dolt-server: lane.
sh_test(
name = "uow_dolt_test",
env = {
"BEADS_TEST_DOLT_SERVER": "local",
"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1",
},
tags = ["dolt-server"],
)
`
if errs := checkDoltServerRules("p", good); len(errs) != 0 {
t.Errorf("good BUILD: %v", errs)
}
for name, bad := range map[string]string{
"no require": strings.Replace(good, " \"BEADS_TEST_REQUIRE_DOLT_CONTAINER\": \"1\",\n },\n tags = [\"dolt-server\"]", " },\n tags = [\"dolt-server\"]", 1),
"container": strings.Replace(good, `"local"`, `"container"`, 1),
"no-remote-exec": strings.Replace(good, `tags = ["dolt-server"]`, `tags = ["dolt-server", "no-remote-exec"]`, 1),
"no target": docker,
"fix switch": strings.Replace(good, `"uow_dolt_test"`, `"fix_dolt_test"`, 1),
"protocol switch": strings.Replace(good, `"uow_dolt_test"`, `"protocol_dolt_test"`, 1),
"proxied switch": strings.Replace(strings.Replace(good, `"uow_dolt_test"`, `"bd_proxied_test"`, 1),
`tags = ["dolt-server"]`, `tags = ["dolt-server-proxied"]`, 1),
"run-dolt switch": strings.Replace(strings.Replace(good, `"uow_dolt_test"`, `"dolt_server_full_test"`, 1),
`tags = ["dolt-server"]`, `tags = ["dolt-server-integration"]`, 1),
"proxied container": strings.Replace(strings.Replace(good, `"local"`, `"container"`, 1),
`tags = ["dolt-server"]`, `tags = ["dolt-server-proxied"]`, 1),
"integration no-remote-exec": strings.Replace(good, `tags = ["dolt-server"]`, `tags = ["dolt-server-integration", "no-remote-exec"]`, 1),
} {
if bad == good {
t.Fatalf("%s: mutation did not apply", name)
}
if errs := checkDoltServerRules("p", bad); len(errs) == 0 {
t.Errorf("%s: accepted", name)
}
}
}
// bazelAttrBlock returns the text of a list attribute (` name = [` up to
// its closing ` ],`) of a rule block from bazelRuleBlock, or "".
func bazelAttrBlock(rule, name string) string {
i := strings.Index(rule, "\n "+name+" = [")
if i < 0 {
return ""
}
end := strings.Index(rule[i:], "\n ],\n")
if end < 0 {
return rule[i:]
}
return rule[i : i+end+7]
}
// bazel-integration mirrors main.yml's integration jobs with the unmodified
// --config=integration (whose .bazelrc filter and flags
// TestBazelrcIntegrationLane pins): no extra filter, selector or remote
// config on the command line, a BEP for the test-count check, and no
// equivalence step. ci and integration share bazel-testlogs (review F3), so
// neither the integration lane in bazel-test nor --config=ci here.
func TestBazelIntegrationJob(t *testing.T) {
workflow := readCIWorkflow(t, bazelWorkflowName)
job := workflow.job(t, bazelIntegJobName)
test := job.step(t, "bazel test //... --config=integration")
// A required PR lane: the step must fit a cold compile (the first
// GitHub run took 17 minutes end to end) followed by the longest test
// action, which .bazelrc caps at its test:integration --test_timeout
// (rbe-west's 1200s limit). The job adds setup and log upload. In mode
// cache it runs on a 4-CPU GitHub-hosted runner, and with the read-only
// cache closed (the farm's kill switch) the whole suite builds and runs
// there cold, about 30 minutes simulated: the step keeps twice that.
const coldCompileMinutes = 20
const cacheClosedMinutes = 30
if test.TimeoutMinutes < 2*cacheClosedMinutes {
t.Errorf("%s test step timeout-minutes = %d, want at least %d (mode cache with the cache closed: ~%d minutes cold on a GitHub-hosted runner)",
bazelIntegJobName, test.TimeoutMinutes, 2*cacheClosedMinutes, cacheClosedMinutes)
}
actionCapMinutes := 0
for line := range bazelrcLines(t) {
if v, ok := strings.CutPrefix(line, "test:integration --test_timeout="); ok {
secs, err := strconv.Atoi(v)
if err != nil || secs <= 0 || secs > 1200 {
t.Fatalf(".bazelrc test:integration --test_timeout=%s, want one value of at most 1200 (rbe-west's cap)", v)
}
actionCapMinutes = (secs + 59) / 60
}
}
if actionCapMinutes == 0 {
t.Fatal(".bazelrc has no test:integration --test_timeout")
}
if want := coldCompileMinutes + actionCapMinutes; test.TimeoutMinutes < want {
t.Errorf("%s test step timeout-minutes = %d, want at least %d (cold compile %d + action cap %d)",
bazelIntegJobName, test.TimeoutMinutes, want, coldCompileMinutes, actionCapMinutes)
}
if job.TimeoutMinutes <= test.TimeoutMinutes || job.TimeoutMinutes > test.TimeoutMinutes+15 {
t.Errorf("%s timeout-minutes = %d, want above the test step's %d by at most 15", bazelIntegJobName, job.TimeoutMinutes, test.TimeoutMinutes)
}
cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(test.Run, " ")
// EXCLUDE_TARGETS: TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers.
const wantCmd = `bazel test //... --config=integration ` + bazelFreshArg + ` --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" --profile="$RUNNER_TEMP/bazel-profile.json" --experimental_build_event_upload_strategy=local ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?`
if !strings.Contains(cmd, wantCmd) || !strings.Contains(test.Run, "set -o pipefail") {
t.Errorf("%s test step does not run exactly %q:\n%s", bazelIntegJobName, wantCmd, test.Run)
}
if n := strings.Count(test.Run, "bazel test //"); n != 1 {
t.Errorf("%s test step runs bazel test %d times, want 1", bazelIntegJobName, n)
}
assertTestStepKeepsExitStatus(t, test)
count := job.step(t, "Every target and shard ran tests")
const wantCount = `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" --not-go //tools/bazel:dolt_version_test --not-go //scripts/repochecks:all`
if strings.TrimSpace(count.Run) != wantCount ||
count.If != "${{ always() && steps.test.outcome != 'skipped' }}" ||
(count.ContinueOnError != nil && count.ContinueOnError != false) {
t.Errorf("test-count step: if=%q continue-on-error=%v run=%q; want run %q", count.If, count.ContinueOnError, count.Run, wantCount)
}
logs := job.step(t, "Upload test logs")
if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != "bazel-integration-testlogs" {
t.Errorf("test-log upload: if=%q name=%q", logs.If, logs.With["name"])
}
for name, j := range workflow.Jobs {
for _, step := range j.Steps {
integ := strings.Contains(step.Run, "--config=integration")
if name == bazelIntegJobName && (strings.Contains(step.Run, "--config=ci") || strings.Contains(step.Run, "equivalence.py")) {
t.Errorf("%s step %q runs the ci lane or equivalence.py; they would read each other's bazel-testlogs", name, step.Name)
}
if name != bazelIntegJobName && integ {
t.Errorf("%s step %q runs --config=integration; only %s may (shared bazel-testlogs)", name, step.Name, bazelIntegJobName)
}
}
}
if err := checkBazelrcIntegrationLane(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc")); err != nil {
t.Error(err)
}
}
// bazelrcLines returns .bazelrc's trimmed lines as a set.
func bazelrcLines(t *testing.T) map[string]bool {
t.Helper()
lines := map[string]bool{}
for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") {
lines[strings.TrimSpace(line)] = true
}
return lines
}
// assertBazelTierStep checks the test step of a remote-only tier job in
// bazel.yml: it runs exactly `bazel test //... --config=<config>` with a BEP,
// keeps bazel's exit status, has a timeout below the job's, and is followed
// by check_testcases.py on that BEP.
func assertBazelTierStep(t *testing.T, job ciWorkflowJob, jobName, config string) {
t.Helper()
const id, bep, wantIf = "test", "bazel-bep.json", ""
step := job.step(t, "bazel test //... --config="+config)
if step.ID != id || step.If != wantIf || step.TimeoutMinutes == 0 || step.TimeoutMinutes >= job.TimeoutMinutes {
t.Errorf("%s step --config=%s: id=%q if=%q timeout-minutes=%d; want id %q, if %q, a timeout below the job's %d",
jobName, config, step.ID, step.If, step.TimeoutMinutes, id, wantIf, job.TimeoutMinutes)
}
cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(step.Run, " ")
wantCmd := `bazel test //... --config=` + config + ` --build_event_json_file="$RUNNER_TEMP/` + bep + `"`
// Every `bazel test` ends its configs with nightly's fresh-results
// switch (ci_merge_queue_test.go).
wantCmd = `bazel test //... --config=` + config + ` ` + bazelFreshArg + ` --build_event_json_file="$RUNNER_TEMP/` + bep + `"`
if !strings.Contains(cmd, wantCmd) || !strings.Contains(step.Run, "set -o pipefail") || strings.Count(step.Run, "bazel test //") != 1 ||
strings.Contains(step.Run, "--config=remote-exec") || strings.Contains(step.Run, "--test_tag_filters") {
t.Errorf("%s step --config=%s does not run exactly %q:\n%s", jobName, config, wantCmd, step.Run)
}
assertTestStepKeepsExitStatus(t, step)
found := false
for _, s := range job.Steps {
if strings.TrimSpace(s.Run) == `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/`+bep+`"` {
found = true
if s.If != "${{ always() && steps."+id+".outcome != 'skipped' }}" || (s.ContinueOnError != nil && s.ContinueOnError != false) {
t.Errorf("%s: check_testcases.py for %s: if=%q continue-on-error=%v", jobName, bep, s.If, s.ContinueOnError)
}
}
}
if !found {
t.Errorf("%s: no check_testcases.py step on %s (a shard or selector that runs no tests exits 0)", jobName, bep)
}
}
// The proxied-server tier (pr-risk.yml "Test (Proxied Dolt Cmd N/15)") and the server-Dolt storage tier (pr-risk.yml "Test (Server Dolt
// Conformance)", "Test (Server Dolt Full Suite N/16)") as Bazel variants:
// each manifest-sharded variant runs its jobs' shard script with their shard
// total, the conformance variant the job's exact flags, with the jobs' race
// setting, subprocess binaries and switches; the lanes' configs keep the
// jobs' selection, and the storage lane shares --config=integration's build.
// bazel.yml runs both tiers remotely only.
func TestBazelDoltServerTiers(t *testing.T) {
rc := bazelrcLines(t)
for _, want := range []string{
"test:doltserver-proxied --@rules_go//go/config:race",
"test:doltserver-proxied --test_tag_filters=dolt-server-proxied",
"test:doltserver-proxied --test_timeout=-1,-1,-1,1200",
"test:doltserver-integration --test_tag_filters=dolt-server-integration",
"test:doltserver-integration --test_timeout=-1,-1,-1,1200",
} {
if !rc[want] {
t.Errorf(".bazelrc lacks %q", want)
}
}
var tierTags, integTags map[string]bool
for line := range rc {
for _, config := range []string{"doltserver-proxied", "doltserver-integration"} {
if strings.HasPrefix(line, "test:"+config+" ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP") ||
strings.Contains(line, "-test.run") || strings.Contains(line, "-test.skip") || strings.Contains(line, "--test_filter")) {
t.Errorf(".bazelrc %q: the jobs run their scripts' selection, without -short or BEADS_TEST_SKIP", line)
}
}
if v, ok := strings.CutPrefix(line, "build:doltserver-integration --@rules_go//go/config:tags="); ok {
tierTags = tagSet(v)
}
if v, ok := strings.CutPrefix(line, "build:integration --@rules_go//go/config:tags="); ok {
integTags = tagSet(v)
}
}
// The same build flags as --config=integration, or the tier compiles the
// tagged graph once more.
if tierTags == nil || !sameTagSet(tierTags, integTags) {
t.Errorf(".bazelrc build:doltserver-integration tags = %v, want build:integration's %v", tierTags, integTags)
}
if rc["test:integration --@rules_go//go/config:race"] != rc["test:doltserver-integration --@rules_go//go/config:race"] {
t.Error(".bazelrc: test:doltserver-integration and test:integration differ in race; they must share their build")
}
// The manifest-sharded tiers PR Risk's retired "Test (Proxied Dolt Cmd
// N/15)" and "Test (Server Dolt Full Suite N/16)" jobs ran
// (ga-96smfk.22): the same shard scripts, test binaries and env, now
// sharded by each target's own BUILD.bazel shard_count
// (TestBazelRetiredLanesCheckListedTestsRan pins it against bazel.yml's
// check_shard_coverage.py argument).
type shardTier struct {
script, binVar, pkg, target, env string
}
tiers := []shardTier{
{".github/scripts/proxied-test-shard.sh", "BEADS_TEST_CMD_BINARY", "cmd/bd", "bd_proxied_test", "BEADS_TEST_PROXIED_SERVER"},
{".github/scripts/server-storage-test-shard.sh", "BEADS_TEST_SERVER_TEST_BINARY", "internal/storage/dolt", "dolt_server_full_test", "BEADS_TEST_ENV_RUN_DOLT"},
}
for _, c := range tiers {
root := sourceRepoRoot(t)
rule := bazelRuleBlock(readPolicyFile(t, root, c.pkg+"/BUILD.bazel"), c.target)
for _, w := range []string{
`srcs = ["//tools/bazel:go_test_manifest_shard.sh"],`,
`"$(rootpath //:` + c.script + `)",`,
`"` + c.binVar + `",`,
`timeout = "eternal",`,
`"` + c.env + `": "1"`,
} {
if !strings.Contains(rule, w) {
t.Errorf("%s:%s does not contain %q:\n%s", c.pkg, c.target, w, rule)
}
}
mm := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script))
if mm == nil {
t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script)
}
data := bazelAttrBlock(rule, "data")
for _, file := range []string{c.script, mm[1]} {
if !strings.Contains(data, `"//:`+file+`",`) {
t.Errorf("%s:%s data lacks //:%s (without the manifest every shard falls back to hashing):\n%s", c.pkg, c.target, file, data)
}
}
}
// PR Risk's retired test-server-storage job's command, frozen.
const conf = `/tmp/dolt-conformance-test -test.v -test.count=1 -test.timeout=15m -test.run '^TestConformance$'`
quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`)
fields := strings.Fields(conf)
if len(fields) == 0 || fields[0] != "/tmp/dolt-conformance-test" {
t.Fatalf("conformance command %q does not run /tmp/dolt-conformance-test", conf)
}
var wantArgs []string
for _, m := range quoted.FindAllStringSubmatch(conf, -1) {
if m[1] != "" {
wantArgs = append(wantArgs, `"`+m[1]+"="+strings.ReplaceAll(m[2], "$", "$$")+`",`)
} else {
wantArgs = append(wantArgs, `"`+m[3]+`",`)
}
}
if len(wantArgs) != 4 {
t.Fatalf("parsed %v from %q", wantArgs, conf)
}
root := sourceRepoRoot(t)
doltBuild := readPolicyFile(t, root, "internal/storage/dolt/BUILD.bazel")
rule := bazelRuleBlock(doltBuild, "dolt_server_conformance_test")
for _, w := range append(wantArgs, `"$(rootpath :dolt_race_off)",`, `srcs = ["//tools/bazel:go_test_variant.sh"],`) {
if !strings.Contains(bazelAttrBlock(rule, "args")+rule, w) {
t.Errorf("dolt:dolt_server_conformance_test does not contain %q (the retired test-server-storage job's selection):\n%s", w, rule)
}
}
if strings.Contains(rule, "BEADS_TEST_ENV_RUN_DOLT") {
t.Error("dolt:dolt_server_conformance_test sets BEADS_TEST_ENV_RUN_DOLT; test-server-storage does not")
}
if r := bazelRuleBlock(doltBuild, "dolt_race_off"); !strings.Contains(r, "go_test_race_off(") || !strings.Contains(r, `test = ":dolt_test",`) {
t.Errorf("dolt:dolt_race_off must be go_test_race_off of :dolt_test (the jobs' binary is not race):\n%s", r)
}
full := bazelRuleBlock(doltBuild, "dolt_server_full_test")
for _, w := range []string{`"$(rootpath :dolt_race_off)",`, `"BEADS_TEST_SUBPROCESS_BINARY": "$(rlocationpath :dolt_race_off)"`, `"//:go.mod",`} {
if !strings.Contains(full, w) {
t.Errorf("dolt:dolt_server_full_test lacks %q (SubprocessRunner reuses the test binary; ModuleRoot needs go.mod):\n%s", w, full)
}
}
proxied := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_proxied_test")
for _, w := range []string{`"$(rootpath :bd_test)",`, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)"`} {
if !strings.Contains(proxied, w) {
t.Errorf("cmd/bd:bd_proxied_test lacks %q (race bd_test, non-race subprocess bd, like the jobs):\n%s", w, proxied)
}
}
for pkg, build := range map[string]string{"cmd/bd": readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "internal/storage/dolt": doltBuild} {
for _, err := range checkDoltServerRules(pkg, build) {
t.Error(err)
}
}
// Each tier is a remote-only job of its own (the job if is pinned by
// TestBazelWorkflowJobsAndExecutionMode via bazelRemoteOnlyJobs, and
// pr.yml's gate requires it: bazelLaneGateIDs): not a step of
// bazel-doltserver (which also runs locally, and whose job a gate may
// require) or of bazel-integration (which a caller may switch off,
// while the server tier always runs remotely).
workflow := readCIWorkflow(t, bazelWorkflowName)
for _, c := range []struct {
job, config, logs string
wantSteps int
}{
// +2 over the earlier counts: the CI analytics summary/upload steps
// (S3; see bazel-test's "CI analytics summary" comment), added to
// every lane job ahead of the result recorder. +1: the remote repo
// contents cache reader (TestBazelRRCReadSteps).
{bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 11},
// One more than bazel-proxied: a "Shard balance" step (rbe-ci-cost-
// latency-study.md recommendation 4), since this is the tier whose
// last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs.
{bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 12},
} {
job := workflow.job(t, c.job)
if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 {
t.Errorf("%s timeout-minutes = %d; it runs remotely only (longest shard ~2-8 min), keep it at most 30", c.job, job.TimeoutMinutes)
}
assertBazelTierStep(t, job, c.job, c.config)
if n := len(job.Steps); n != c.wantSteps {
t.Errorf("%s has %d steps, want %d (checkout, the rrc reader, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps)
}
logs := job.step(t, "Upload test logs")
if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") {
t.Errorf("%s test-log upload: if=%q name=%q uses=%q", c.job, logs.If, logs.With["name"], logs.Uses)
}
}
if serverStorage := workflow.job(t, bazelServerJobName); true {
budget := serverStorage.step(t, "Shard balance")
if budget.If != "${{ always() && steps.test.outcome != 'skipped' }}" {
t.Errorf("%s Shard balance step if=%q", bazelServerJobName, budget.If)
}
if want := `python3 tools/bazel/shard_budget.py --bep "$RUNNER_TEMP/bazel-bep.json"`; strings.TrimSpace(budget.Run) != want {
t.Errorf("%s Shard balance step runs %q, want %q", bazelServerJobName, strings.TrimSpace(budget.Run), want)
}
}
for name, j := range workflow.Jobs {
for _, step := range j.Steps {
proxied := strings.Contains(step.Run, "--config=doltserver-proxied")
server := strings.Contains(step.Run, "--config=doltserver-integration")
if (proxied && name != bazelProxiedJobName) || (server && name != bazelServerJobName) {
t.Errorf("%s step %q runs a dolt-server tier outside its own job", name, step.Name)
}
if (name == bazelIntegJobName || name == bazelDoltJobName) && (proxied || server || strings.Contains(step.Run, "doltserver-")) {
t.Errorf("%s step %q runs the proxied or server storage tier; each has its own job", name, step.Name)
}
}
}
}
// bazel-embedded replaces pr-risk.yml's embedded-Dolt tier: --config=embedded
// runs the embedded variants race like the jobs' binaries, with the jobs'
// parallelism; each shard variant runs its job's shard script with the job's
// shard total, and the conformance variants pass the jobs' exact selectors.
// bazel-embedded's test step, exactly (review F4).
const bazelEmbeddedTestRun = `set -o pipefail
start=$(date +%s)
rc=0
bazel test //... --config=embedded ${BAZEL_FRESH:+"$BAZEL_FRESH"} \
--build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \
--execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \
--profile="$RUNNER_TEMP/bazel-profile.json" \
--experimental_build_event_upload_strategy=local \
2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?
echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY"
exit "$rc"`
// .bazelrc's --config=embedded, exactly and in order (review F4): no
// --test_filter, no -test.short/-test.run/-test.skip, no retries (result
// caching as every lane's: nightly re-executes). The conformance targets' own -test.run/-test.skip args
// (the legacy jobs' partition) are checked against pr-risk.yml below.
var bazelEmbeddedRCLines = []string{
"test:embedded --@rules_go//go/config:race",
"test:embedded --test_tag_filters=embedded",
"test:embedded --build_tests_only",
"test:embedded --keep_going",
"test:embedded --test_summary=terse",
"test:embedded --test_timeout=-1,-1,-1,1200",
"test:embedded --test_arg=-test.parallel=4",
"test:embedded --test_env=GO_TEST_WRAP_TESTV=1",
"test:embedded --remote_download_regex=.*/test\\.xml$",
"test:embedded --experimental_remote_cache_eviction_retries=0",
// F5 S1: the race configuration's nogo is owned by the test lane.
"test:embedded --norun_validations",
}
func TestBazelEmbeddedJobRunsEmbeddedTier(t *testing.T) {
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName)
test := job.step(t, "bazel test //... --config=embedded")
if !strings.Contains(test.Run, "bazel test //... --config=embedded") || !strings.Contains(test.Run, "set -o pipefail") {
t.Errorf("embedded step does not run --config=embedded over //...:\n%s", test.Run)
}
if strings.Contains(test.Run, "--config=remote-exec") {
t.Errorf("embedded step selects remote-exec itself; setup-bazel's rc does that only when secrets are present")
}
assertTestStepKeepsExitStatus(t, test)
// Review F4: the exact command line. Since D2 step 1 this lane is the
// tier's only pre-merge run on same-repo PRs, so an extra flag (a
// --test_filter, a --test_arg=-test.short/-test.run/-test.skip) that
// quietly narrows it must be a reviewed edit of this test too.
if strings.TrimSpace(test.Run) != bazelEmbeddedTestRun {
t.Errorf("embedded step run changed; want exactly:\n%s\ngot:\n%s", bazelEmbeddedTestRun, test.Run)
}
// A shard with no tests assigned, or a selector that matches nothing,
// exits 0; the job fails on any target or shard whose test.xml lists none.
if !strings.Contains(test.Run, `--build_event_json_file="$RUNNER_TEMP/bazel-bep.json"`) {
t.Errorf("embedded step writes no BEP for the test-count check:\n%s", test.Run)
}
count := job.step(t, "Every target and shard ran tests")
if strings.TrimSpace(count.Run) != `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json"` ||
count.If != "${{ always() && steps.test.outcome != 'skipped' }}" ||
(count.ContinueOnError != nil && count.ContinueOnError != false) {
t.Errorf("test-count step: if=%q continue-on-error=%v run=%q", count.If, count.ContinueOnError, count.Run)
}
rc := map[string]bool{}
for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") {
rc[strings.TrimSpace(line)] = true
}
for _, want := range []string{
"test:embedded --@rules_go//go/config:race",
"test:embedded --test_tag_filters=embedded",
// The jobs pass no -parallel: GOMAXPROCS on 4-vCPU ubuntu-latest.
"test:embedded --test_arg=-test.parallel=4",
} {
if !rc[want] {
t.Errorf(".bazelrc lacks %q", want)
}
}
// Review F4: --config=embedded is exactly these lines, in this order, and
// no unconfigured common/build/test line (which applies to every config)
// selects or narrows tests.
var embeddedLines []string
narrow := regexp.MustCompile(`test_filter|test_arg|-test\.(short|run|skip)|test_tag_filters|test_lang_filters|test_size_filters|test_timeout_filters`)
for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") {
line = strings.TrimSpace(line)
cmd, _, _ := strings.Cut(line, " ")
if strings.HasSuffix(cmd, ":embedded") {
embeddedLines = append(embeddedLines, line)
}
if (cmd == "common" || cmd == "build" || cmd == "test") && narrow.MatchString(line) {
t.Errorf(".bazelrc %q narrows every config's tests, including --config=embedded", line)
}
}
if !reflect.DeepEqual(embeddedLines, bazelEmbeddedRCLines) {
t.Errorf(".bazelrc --config=embedded lines changed; want exactly:\n%s\ngot:\n%s",
strings.Join(bazelEmbeddedRCLines, "\n"), strings.Join(embeddedLines, "\n"))
}
if gen := readPolicyFile(t, sourceRepoRoot(t), setupBazelActionDir+"/write-bazelrc.sh"); narrow.MatchString(gen) {
t.Errorf("setup-bazel's generated rc selects or narrows tests; it may only configure remote execution")
}
for line := range rc {
// Result caching: only nightly's fresh config turns it off
// (ci_merge_queue_test.go).
if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && line != bazelFreshRCLine {
t.Errorf(".bazelrc %q: only test:fresh sets test result caching", line)
}
if strings.HasPrefix(line, "test:embedded ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP")) {
t.Errorf(".bazelrc %q: the embedded jobs run without -short and BEADS_TEST_SKIP", line)
}
}
// PR Risk's retired embedded tier (ga-96smfk.22): its shard scripts,
// conformance partitions and env, which the Bazel targets below run.
sharded := []struct {
job, script, pkg, target string
}{
{"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd", "bd_embedded_test"},
{"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd", "bd_embedded_part2_test"},
{"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt", "embeddeddolt_embedded_test"},
}
for _, c := range sharded {
root := sourceRepoRoot(t)
rule := bazelRuleBlock(readPolicyFile(t, root, c.pkg+"/BUILD.bazel"), c.target)
// c.target's own shard_count is the lane's source of truth;
// TestBazelRetiredLanesCheckListedTestsRan ties it to bazel.yml's
// check_shard_coverage.py argument.
for _, want := range []string{
`srcs = ["//tools/bazel:go_test_manifest_shard.sh"],`,
`"$(rootpath //:` + c.script + `)",`,
`"BEADS_TEST_EMBEDDED_DOLT": "1"`,
`"embedded"`,
} {
if !strings.Contains(rule, want) {
t.Errorf("%s:%s does not contain %q (%s):\n%s", c.pkg, c.target, want, c.job, rule)
}
}
// The script's -test.timeout=20m equals Bazel's 1200s action limit,
// which kills without a goroutine dump; 19m lets Go's fire first.
script := readPolicyFile(t, root, c.script)
if !strings.Contains(script, "-test.timeout=20m") {
t.Errorf("%s no longer passes -test.timeout=20m; revisit %s:%s's -test.timeout=19m", c.script, c.pkg, c.target)
}
if !strings.Contains(bazelAttrBlock(rule, "args"), `"-test.timeout=19m",`) {
t.Errorf("%s:%s must pass -test.timeout=19m after the script's own:\n%s", c.pkg, c.target, rule)
}
// The script and its manifest must be in the runfiles: without the
// manifest the script silently falls back to hash assignment, so
// every shard still passes but no longer runs its job's tests.
m := shardManifestDefault.FindStringSubmatch(script)
if m == nil {
t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script)
}
data := bazelAttrBlock(rule, "data")
for _, file := range []string{c.script, m[1]} {
if !strings.Contains(data, `"//:`+file+`",`) {
t.Errorf("%s:%s data lacks //:%s:\n%s", c.pkg, c.target, file, data)
}
}
}
// The cmd jobs' subprocess bd is the race build, as //cmd/bd:bd is
// under --config=embedded (bd_for_tests never is).
for _, name := range embeddedCmdTargets {
rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "cmd/bd/BUILD.bazel"), name)
if !strings.Contains(rule, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)"`) {
t.Errorf("cmd/bd:%s must run the race //cmd/bd:bd as BEADS_TEST_BD_BINARY:\n%s", name, rule)
}
}
// The retired test-embedded-conformance job's two partitions, frozen.
// Core runs as two targets: embeddeddolt_conformance_core_test runs the
// job's core selector but also skips conformanceCoreSlowGroups, which
// embeddeddolt_conformance_core_slow_test runs instead, so between them
// they run every core group exactly once.
conformance := map[string]string{
"core": `/tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$' -test.skip '^TestConformance$/^Audit$'`,
"audit": `/tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$/^Audit$'`,
}
slow := strings.Join(conformanceCoreSlowGroups, "|")
// What each target's rule must say instead of the job's own selector.
retarget := map[string]map[string]string{
"embeddeddolt_conformance_core_test": {
`"-test.skip=^TestConformance$$/^Audit$$",`: `"-test.skip=^TestConformance$$/^(Audit|` + slow + `)$$",`,
},
"embeddeddolt_conformance_core_slow_test": {
`"-test.run=^TestConformance$$",`: `"-test.run=^TestConformance$$/^(` + slow + `)$$",`,
`"-test.skip=^TestConformance$$/^Audit$$",`: "",
},
}
quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`)
for _, c := range []struct{ partition, target string }{
{"core", "embeddeddolt_conformance_core_test"},
{"core", "embeddeddolt_conformance_core_slow_test"},
{"audit", "embeddeddolt_conformance_audit_test"},
} {
partition, target := c.partition, c.target
run := conformance[partition]
fields := strings.Fields(run)
if len(fields) == 0 || fields[0] != "/tmp/embeddeddolt-test" {
t.Fatalf("%s conformance does not run /tmp/embeddeddolt-test: %q", partition, run)
}
var want []string
for _, m := range quoted.FindAllStringSubmatch(run, -1) {
var w string
switch {
case m[1] != "":
w = `"` + m[1] + "=" + strings.ReplaceAll(m[2], "$", "$$") + `",`
case strings.HasPrefix(m[3], "-test.timeout="):
// Documented deviation: Bazel kills at 1200s without a
// goroutine dump, so the variant's Go timeout is 19m.
w = `"-test.timeout=19m",`
default:
w = `"` + m[3] + `",`
}
if r, ok := retarget[target][w]; ok {
w = r
}
if w != "" {
want = append(want, w)
}
}
if len(want) < 4 {
t.Fatalf("parsed only %v from %s conformance %q", want, partition, run)
}
rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "internal/storage/embeddeddolt/BUILD.bazel"), target)
for _, w := range append(want, `"BEADS_TEST_EMBEDDED_DOLT": "1"`, `"embedded"`) {
if !strings.Contains(rule, w) {
t.Errorf("embeddeddolt:%s does not contain %q (the retired %s conformance partition):\n%s", target, w, partition, rule)
}
}
}
}
// conformanceCoreSlowGroups are the conformance.RunAll groups
// embeddeddolt_conformance_core_slow_test runs and
// embeddeddolt_conformance_core_test skips. The core target skips only these
// names, so a group renamed or added in RunAll still runs there: a name here
// that RunAll no longer has matches nothing in either target and loses no
// group, only the split (core gets slower again).
var conformanceCoreSlowGroups = []string{"ReadyCountsPageChunking", "Portable"}
// bazel-pure replaces pr.yml's check-cmd-bd-puregeo-tests job: the same pure
// cmd/bd test selector, the same pure build set, and the js/wasm hook test
// under the same exact-count guard.
// bazel.yml's pure-Go and js/wasm lane is the only run of the CGO_ENABLED=0
// portability boundaries pr.yml's retired check-cmd-bd-puregeo-tests job
// owned (ga-96smfk.22): the same -short cmd/bd subset, the same cmd/bd and
// test-binary builds, and the exact js/wasm refusal contract with its count
// guard. The selectors below are that job's, frozen.
func TestBazelPureJobMirrorsPureGoJob(t *testing.T) {
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelPureJobName)
const pureCmdBDTests = "^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)"
if job.Env["PURE_CMD_BD_TESTS"] != pureCmdBDTests {
t.Errorf("%s PURE_CMD_BD_TESTS = %q, want the pure cmd/bd -run selector %q", bazelPureJobName, job.Env["PURE_CMD_BD_TESTS"], pureCmdBDTests)
}
run := job.step(t, "Run pure-Go cmd/bd test subset (--config=pure)").Run
for _, required := range []string{"bazel test --config=pure " + bazelSoleRunArg + " " + bazelFreshArg + " //cmd/bd:bd_test", `"--test_arg=-test.run=$PURE_CMD_BD_TESTS"`, "(( n > 0 ))"} {
if !strings.Contains(run, required) {
t.Errorf("pure subset step does not contain %q:\n%s", required, run)
}
}
if !strings.Contains(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "test:pure --test_arg=-test.short") {
t.Error(".bazelrc test:pure does not pass -test.short like the pr.yml subset")
}
build := job.step(t, "Build cmd/bd and pure-Go test binaries (--config=pure)").Run
for _, target := range []string{
"//cmd/bd:bd",
"//cmd/bd:bd_test",
"//internal/storage/embeddeddolt:embeddeddolt_test",
"//internal/tracker:tracker_test",
} {
if !strings.Contains(build, target) || !strings.Contains(build, "bazel build --config=pure") {
t.Errorf("pure build step does not build %s:\n%s", target, build)
}
}
// `go build` rejects a pure binary that imports gozstd at compile time;
// Bazel compiles gozstd's stubs, whose init panics, so the lane must start
// every pure artifact (bd_test starts in the subset step above).
start := job.step(t, "Start every pure-Go artifact (gozstd contamination check)").Run
for _, required := range []string{
"set -euo pipefail",
"bazel run --config=pure //cmd/bd:bd -- version",
"bazel test --config=pure",
"//internal/storage/embeddeddolt:embeddeddolt_test",
"//internal/tracker:tracker_test",
"'--test_arg=-test.run=^$'",
} {
if !strings.Contains(start, required) {
t.Errorf("pure artifact start step does not contain %q:\n%s", required, start)
}
}
patch := readPolicyFile(t, sourceRepoRoot(t), "third_party/patches/gozstd_nocgo.patch")
if !strings.Contains(patch, "+func init() { panic(") {
t.Error("gozstd_nocgo.patch stubs no longer panic in init; a contaminated pure binary would start")
}
wasm := job.step(t, "Run js/wasm hook boundary").Run
for _, required := range []string{"bazel build --config=js-wasm //internal/hooks:hooks_test", "go_js_wasm_exec"} {
if !strings.Contains(wasm, required) {
t.Errorf("js/wasm step does not contain %q:\n%s", required, wasm)
}
}
// Exactly one run and one pass of the refusal contract, no failure or
// skip: compiling the package alone would not execute it.
for _, required := range []string{
" -test.run '^TestRunHookReportsUnsupportedExecution$'",
"run_count=0",
`=== RUN TestRunHookReportsUnsupportedExecution`,
`^--- PASS: TestRunHookReportsUnsupportedExecution`,
`nonpass_pattern='^[[:space:]]*--- (FAIL|SKIP): '`,
`[[ "$line" =~ $nonpass_pattern ]]`,
`run_count != 1 || pass_count != 1 || nonpass_count != 0`,
} {
if !strings.Contains(wasm, required) {
t.Errorf("js/wasm step does not contain %q:\n%s", required, wasm)
}
}
}
// swallowedExit matches shell that hides a failing command's status.
var swallowedExit = regexp.MustCompile(`\bexit\b|\|\|\s*(true|:)|;\s*(true|:)\s*$|\bset\s+\+e\b|\btrap\b`)
// shellCommands joins backslash continuations and drops blank and comment
// lines.
func shellCommands(run string) []string {
var cmds []string
cur := ""
for _, line := range strings.Split(run, "\n") {
trimmed := strings.TrimSpace(line)
if cur == "" && (trimmed == "" || strings.HasPrefix(trimmed, "#")) {
continue
}
if strings.HasSuffix(trimmed, "\\") {
cur += strings.TrimSuffix(trimmed, "\\") + " "
continue
}
cmds = append(cmds, strings.Join(strings.Fields(cur+trimmed), " "))
cur = ""
}
if cur != "" {
cmds = append(cmds, strings.Join(strings.Fields(cur), " "))
}
return cmds
}
// A report step's status is its script's: the script is the last command,
// nothing after it on that line, and nothing in the step can mask a failure.
func assertReportStepKeepsExitStatus(t *testing.T, step ciWorkflowStep, invocation string) {
t.Helper()
if step.Shell != "" {
t.Errorf("step %q overrides shell %q; the default bash -e is part of the contract", step.Name, step.Shell)
}
cmds := shellCommands(step.Run)
if len(cmds) == 0 {
t.Errorf("step %q runs nothing", step.Name)
return
}
last := cmds[len(cmds)-1]
if !strings.HasPrefix(last, invocation+" ") || strings.ContainsAny(last, ";&|") {
t.Errorf("step %q must end with a bare %q so its exit status is the step's; last command %q", step.Name, invocation, last)
}
for _, cmd := range cmds {
if swallowedExit.MatchString(cmd) {
t.Errorf("step %q command %q can hide the script's exit status", step.Name, cmd)
}
}
}
// The bazel test step captures bazel's status through tee and must exit
// with it.
func assertTestStepKeepsExitStatus(t *testing.T, step ciWorkflowStep) {
t.Helper()
if step.Shell != "" {
t.Errorf("step %q overrides shell %q", step.Name, step.Shell)
}
cmds := shellCommands(step.Run)
if len(cmds) == 0 || cmds[len(cmds)-1] != `exit "$rc"` {
t.Errorf("step %q must end with exit \"$rc\"; commands %q", step.Name, cmds)
}
exits := 0
for _, cmd := range cmds {
if regexp.MustCompile(`(^|[;&|]\s*)exit\b`).MatchString(cmd) {
exits++
}
if regexp.MustCompile(`\|\|\s*(true|:)|;\s*(true|:)\s*$|\bset\s+\+e\b|\btrap\b`).MatchString(cmd) {
t.Errorf("step %q command %q can hide bazel's exit status", step.Name, cmd)
}
}
if exits != 1 {
t.Errorf("step %q has %d exit commands, want only the final exit \"$rc\"", step.Name, exits)
}
}
func readYAMLNode(t *testing.T, rel string) *yaml.Node {
t.Helper()
path := filepath.Join(sourceRepoRoot(t), rel)
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var doc yaml.Node
if err := yaml.Unmarshal(data, &doc); err != nil {
t.Fatalf("parse %s: %v", path, err)
}
if doc.Kind != yaml.DocumentNode || len(doc.Content) != 1 || doc.Content[0].Kind != yaml.MappingNode {
t.Fatalf("%s: want a YAML mapping document", path)
}
return doc.Content[0]
}
// walkYAML calls fn with the dotted path of every mapping key and scalar.
func walkYAML(node *yaml.Node, path string, fn func(path string, key bool, value string)) {
switch node.Kind {
case yaml.MappingNode:
for i := 0; i+1 < len(node.Content); i += 2 {
k := node.Content[i].Value
fn(path+"."+k, true, k)
walkYAML(node.Content[i+1], path+"."+k, fn)
}
case yaml.SequenceNode:
for i, item := range node.Content {
walkYAML(item, fmt.Sprintf("%s[%d]", path, i), fn)
}
case yaml.ScalarNode:
fn(path, false, node.Value)
case yaml.AliasNode:
if node.Alias != nil {
walkYAML(node.Alias, path, fn)
}
}
}
func yamlMapKeys(node *yaml.Node, key string) []string {
for i := 0; i+1 < len(node.Content); i += 2 {
if node.Content[i].Value != key {
continue
}
v := node.Content[i+1]
switch v.Kind {
case yaml.MappingNode:
var keys []string
for j := 0; j < len(v.Content); j += 2 {
keys = append(keys, v.Content[j].Value)
}
return keys
case yaml.SequenceNode:
var keys []string
for _, item := range v.Content {
keys = append(keys, item.Value)
}
return keys
default:
return []string{v.Value}
}
}
return nil
}
// bazel.yml hands the RBE credentials to PR-head code on same-repo PRs (an
// accepted, documented risk). Keep that exposure from growing: only the
// listed triggers (never pull_request_target), secrets referenced from the
// setup-bazel step's env and nowhere else (not workflow/job env, run, with or
// if), and no continue-on-error anywhere, so a red lane stays red.
func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) {
root := readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName))
triggers := yamlMapKeys(root, "on")
sort.Strings(triggers)
if !reflect.DeepEqual(triggers, bazelWorkflowTriggers) {
t.Errorf("%s triggers = %v, want exactly %v", bazelWorkflowName, triggers, bazelWorkflowTriggers)
}
setupSteps := map[string]bool{}
for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs {
if name == bazelRBEJobName || name == bazelRBEPrewarmJobName {
continue // no setup-bazel; rbe-prewarm's secret reads are pinned below
}
n := 0
for i, step := range job.Steps {
if step.Uses == "./"+setupBazelActionDir {
setupSteps[fmt.Sprintf(".jobs.%s.steps[%d].env.", name, i)] = true
n++
}
}
if n != 1 {
t.Errorf("%s job %s has %d setup-bazel steps, want 1", bazelWorkflowName, name, n)
}
}
ciAnalyticsPaths := ciAnalyticsContinueOnErrorPaths(t)
secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`)
walkYAML(root, "", func(path string, key bool, value string) {
if key && value == "continue-on-error" {
// rbe-prewarm's job-level continue-on-error is a deliberate
// exception (TestBazelWorkflowJobsAndExecutionMode's comment on
// the job has the full reasoning): without it, this advisory
// job's failure would still fail bazel.yml's own run and
// cascade into pr.yml's ci-gate. So is each flag-gated lane's
// (bazelFlagGatedLanes): advisory until its pr.yml flag is
// "true", and required then through its own gate id. So are the
// two CI analytics steps in every lane job
// (bazelCIAnalyticsStepNames): reporting only, never a `needs`
// of anything and never read back by this workflow or its
// callers.
// And each shadow lane's (bazelShadowLanes): it reports into no
// gate, so its failure must not fail the call either.
_, flagGated := bazelFlagGatedLanes[strings.TrimSuffix(strings.TrimPrefix(path, ".jobs."), ".continue-on-error")]
_, shadow := bazelShadowLanes[strings.TrimSuffix(strings.TrimPrefix(path, ".jobs."), ".continue-on-error")]
if path != bazelRBEPrewarmContinueOnErrorPath && !flagGated && !shadow && !ciAnalyticsPaths[path] {
t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path)
}
}
if key && (value == "secrets" && strings.HasPrefix(path, ".jobs.")) {
t.Errorf("%s: %s passes secrets to a called workflow or container", bazelWorkflowName, path)
}
if key || !secretRef.MatchString(value) {
return
}
if path == bazelRBESecretPath && value == bazelRBESecretValue {
return // the rbe job's emptiness test (TestBazelRBEJobDecidesOnce)
}
switch {
case path == bazelRBEPrewarmHasAppPath && value == bazelRBEPrewarmHasAppValue,
path == bazelRBEPrewarmKeyPath && value == bazelRBEPrewarmKeyValue:
return // rbe-prewarm's own credential-check/mint step (see their doc comment)
}
prefix := path[:strings.LastIndex(path, ".")+1]
if !setupSteps[prefix] {
t.Errorf("%s: %s reads secrets (%q); only the setup-bazel step's env may", bazelWorkflowName, path, value)
}
})
action := readYAMLNode(t, filepath.Join(setupBazelActionDir, "action.yml"))
walkYAML(action, "", func(path string, key bool, value string) {
if key && value == "continue-on-error" {
t.Errorf("setup-bazel: %s hides failures", path)
}
if !key && secretRef.MatchString(value) {
t.Errorf("setup-bazel: %s reads secrets directly; the caller passes them in the step env", path)
}
})
}
// The execution mode (remote, cache, local, skip) is decided once, by the rbe job,
// so a run cannot mix modes and no lane can drift from the others (review D1
// F1/F5/F6): Dependabot PRs are same-repo but get no secrets, so a condition
// on vars and fork alone sent them to the remote-only path with nothing to
// run it. The job's decide step (its first) reads nothing but booleans, runs
// no repository code, and its outputs are the job's and the workflow_call
// outputs; the steps after it are the worker-env preflight
// (TestBazelRBEWorkerEnvPreflight), which set no output.
func TestBazelRBEJobDecidesOnce(t *testing.T) {
workflow := readCIWorkflow(t, bazelWorkflowName)
job := workflow.job(t, bazelRBEJobName)
// F3: runs-on picks the *venue* (Blacksmith for trusted same-repo PRs,
// ubuntu-latest otherwise) - a separate concern from the RBE mode
// decision (remote/cache/local/skip), which still lives solely in the
// decide step's outputs below.
if len(job.Needs) != 0 || job.If != "" || job.RunsOn != wantRBERunsOn || len(job.Env) != 0 {
t.Errorf("%s: needs %v, if %q, runs-on %q, env %v; want no needs, if or env, on %q",
bazelRBEJobName, job.Needs, job.If, job.RunsOn, job.Env, wantRBERunsOn)
}
wantOutputs := map[string]string{
"enabled": "${{ steps.decide.outputs.enabled }}",
"mode": "${{ steps.decide.outputs.mode }}",
"tier": "${{ steps.decide.outputs.tier }}",
// The remote repo contents cache's switch, from decide's mode
// (TestBazelRRCModeStep).
"rrc": "${{ steps.rrc.outputs.rrc }}",
}
if !reflect.DeepEqual(job.Outputs, wantOutputs) {
t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs)
}
if len(job.Steps) != 4 {
t.Fatalf("%s has %d steps, want the decision step, the rrc step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps))
}
step := job.Steps[0]
wantEnv := map[string]string{
"RBE_VAR_ON": "${{ vars.RBE_WEST_WORKERS == 'true' }}",
"RBE_INPUT_OFF": "${{ inputs.rbe == 'off' }}",
"RBE_INPUT_CACHE": "${{ inputs.rbe == 'cache' }}",
"PULL_REQUEST": prRiskPullRequestValue,
"FORK": "${{ github.event.pull_request.head.repo.fork == true }}",
"DEPENDABOT": prRiskDependabotValue,
"FORK_FARM": bazelForkFarmValue,
"HAS_EXECUTOR": bazelRBESecretValue,
"PR_NUMBER": "${{ github.event.pull_request.number }}",
}
if step.ID != "decide" || step.Uses != "" || step.Shell != "" || len(step.With) != 0 || !reflect.DeepEqual(step.Env, wantEnv) {
t.Errorf("%s step: id %q, uses %q, shell %q, with %v, env %v; want id decide, a run step with env %v",
bazelRBEJobName, step.ID, step.Uses, step.Shell, step.With, step.Env, wantEnv)
}
// The workflow_call outputs are the rbe job's, for a caller's gate.
var doc struct {
On struct {
WorkflowCall struct {
Outputs map[string]struct {
Value string `yaml:"value"`
} `yaml:"outputs"`
} `yaml:"workflow_call"`
} `yaml:"on"`
}
if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName)), &doc); err != nil {
t.Fatal(err)
}
// (The lanes' result outputs: TestBazelLaneIsGatedAlongsideLegacy.)
for name, want := range map[string]string{
"rbe-enabled": "${{ jobs.rbe.outputs.enabled }}",
"rbe-mode": "${{ jobs.rbe.outputs.mode }}",
} {
if got := doc.On.WorkflowCall.Outputs[name].Value; got != want {
t.Errorf("workflow_call output %s = %q, want %q", name, got, want)
}
}
// Nothing outside the decision step re-derives the condition.
rederive := regexp.MustCompile(`(?i)vars\.RBE_WEST_WORKERS|inputs\.rbe\b|inputs\.fork-farm|head\.repo\.fork|github\.actor|dependabot`)
walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "", func(path string, key bool, value string) {
// The decision step itself (its env and, since rbe-fork, its run:
// the DEPENDABOT fact) is the one place that may.
if key || !rederive.MatchString(value) || strings.HasPrefix(path, ".jobs."+bazelRBEJobName+".steps[0].env.") ||
path == ".jobs."+bazelRBEJobName+".steps[0].run" {
return
}
// F3: the rbe job's own runs-on picks a runner venue (Blacksmith vs
// ubuntu-latest) for trusted same-repo PRs; it shares some of the
// same trust predicates (github.actor, dependabot) as the mode
// decision but decides something else entirely (where the decide
// step runs, not what it decides) - not a re-derivation of mode.
if path == ".jobs."+bazelRBEJobName+".runs-on" {
return
}
// The checkout opt-in for fork code (TestBazelWorkflowForkFarmInputs
// pins it): a checkout input, not an execution-mode decision.
if value == bazelAllowUnsafeCheckout && strings.HasSuffix(path, ".with.allow-unsafe-pr-checkout") {
return
}
t.Errorf("%s: %s re-derives the execution mode (%q); read needs.rbe.outputs instead", bazelWorkflowName, path, value)
})
// The dispatch choices: on, cache (fork simulation) and off.
var dispatch struct {
On struct {
WorkflowDispatch struct {
Inputs map[string]struct {
Options []string `yaml:"options"`
} `yaml:"inputs"`
} `yaml:"workflow_dispatch"`
} `yaml:"on"`
}
if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName)), &dispatch); err != nil {
t.Fatal(err)
}
if got, want := dispatch.On.WorkflowDispatch.Inputs["rbe"].Options, []string{"on", "cache", "off"}; !reflect.DeepEqual(got, want) {
t.Errorf("workflow_dispatch rbe options = %v, want %v", got, want)
}
// The decision, for the facts GitHub evaluates the env expressions on
// (its == is case-insensitive; a missing secret reads as ''), and for
// fork and Dependabot pull_request runs what rbe-fork-mint's /v1/status
// answers (runBazelRBEDecision's curl stub; "" = unreachable).
type facts struct {
rbeVar, rbeInput, secret string
fork, farm bool
pr, dependabot bool // event pull_request; github.actor dependabot[bot]
mint string
}
cases := []struct {
name string
in facts
mode, enabled, tier string
}{
{"same-repo PR with secrets", facts{"true", "", "grpcs://x", false, false, true, false, ""}, "remote", "true", ""},
{"push to main", facts{"true", "", "grpcs://x", false, false, false, false, ""}, "remote", "true", ""},
{"var in other case", facts{"True", "on", "grpcs://x", false, false, false, false, ""}, "remote", "true", ""},
// Fork and Dependabot pull_request runs: rbe-fork decides. Open:
// remote execution with a mint certificate, tier ro or rw. Closed,
// refusing, unreachable or nonsense: local execution with the
// read-only cache, as before rbe-fork (rbe=cache simulates them);
// rbe=off drops the cache too.
{"fork PR, mint ro", facts{"true", "", "", true, false, true, false, "ro"}, "fork-ro", "true", "ro"},
{"fork PR, mint rw", facts{"true", "", "", true, false, true, false, "rw"}, "fork-rw", "true", "rw"},
{"fork PR, mint ro, var unset (forks cannot read it)", facts{"", "", "", true, false, true, false, "ro"}, "fork-ro", "true", "ro"},
{"fork PR somehow with the secret, mint ro", facts{"true", "", "grpcs://x", true, false, true, false, "ro"}, "fork-ro", "true", "ro"},
{"Dependabot PR, mint ro", facts{"true", "", "", false, false, true, true, "ro"}, "fork-ro", "true", "ro"},
{"Dependabot PR, var unset, mint ro", facts{"", "", "", false, false, true, true, "ro"}, "fork-ro", "true", "ro"},
{"fork PR, mint unreachable", facts{"true", "", "", true, false, true, false, ""}, "cache", "false", ""},
{"fork PR, mint closed", facts{"true", "", "", true, false, true, false, "closed"}, "cache", "false", ""},
{"fork PR, rw tier closed", facts{"true", "", "", true, false, true, false, "rw-closed"}, "cache", "false", ""},
{"fork PR, canary refuses", facts{"true", "", "", true, false, true, false, "canary"}, "cache", "false", ""},
{"fork PR, mint busy", facts{"true", "", "", true, false, true, false, "busy"}, "cache", "false", ""},
{"fork PR, mint answers garbage", facts{"true", "", "", true, false, true, false, "garbage"}, "cache", "false", ""},
{"fork PR, mint answers an unknown tier", facts{"true", "", "", true, false, true, false, "evil"}, "cache", "false", ""},
{"fork PR, var unset, mint unreachable", facts{"", "", "", true, false, true, false, ""}, "cache", "false", ""},
{"fork PR, var false, mint unreachable", facts{"false", "", "", true, false, true, false, ""}, "cache", "false", ""},
{"Dependabot PR, mint unreachable", facts{"true", "", "", false, false, true, true, ""}, "cache", "false", ""},
{"Dependabot, var unset, mint unreachable", facts{"", "", "", false, false, true, true, ""}, "cache", "false", ""},
{"dispatch rbe=cache", facts{"true", "cache", "grpcs://x", false, false, false, false, ""}, "cache", "false", ""},
{"dispatch rbe=cache, var unset", facts{"", "cache", "", false, false, false, false, ""}, "cache", "false", ""},
{"call rbe=CACHE", facts{"true", "CACHE", "grpcs://x", false, false, true, false, ""}, "cache", "false", ""},
{"fork PR rbe=cache, mint ro", facts{"true", "cache", "", true, false, true, false, "ro"}, "cache", "false", ""},
{"dispatch rbe=off", facts{"true", "off", "grpcs://x", false, false, false, false, ""}, "local", "false", ""},
{"call rbe=OFF", facts{"true", "OFF", "grpcs://x", false, false, true, false, ""}, "local", "false", ""},
{"fork PR rbe=off, mint ro", facts{"true", "off", "", true, false, true, false, "ro"}, "local", "false", ""},
{"same-repo, var unset", facts{"", "", "grpcs://x", false, false, true, false, ""}, "skip", "false", ""},
{"same-repo without the secret", facts{"true", "", "", false, false, true, false, ""}, "cache", "false", ""},
{"var false", facts{"false", "on", "grpcs://x", false, false, false, false, ""}, "skip", "false", ""},
// bazel-farm.yml's authorized fork runs: remote or nothing, never
// a second local run (pr.yml already runs one). Not a pull_request
// event, so the mint is never asked.
{"authorized fork farm", facts{"true", "", "grpcs://x", true, true, false, false, "ro"}, "remote", "true", ""},
{"authorized fork farm, var unset", facts{"", "", "grpcs://x", true, true, false, false, ""}, "skip", "false", ""},
{"authorized fork farm, no secret", facts{"true", "", "", true, true, false, false, ""}, "skip", "false", ""},
{"authorized fork farm, rbe=off", facts{"true", "off", "grpcs://x", true, true, false, false, ""}, "local", "false", ""},
// A fork run that is neither a pull_request nor authorized
// (pull_request_target without bazel-farm.yml's inputs) stays local.
{"fork, not authorized, secret", facts{"true", "", "grpcs://x", true, false, false, false, "ro"}, "cache", "false", ""},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
env := map[string]string{
"RBE_VAR_ON": strconv.FormatBool(strings.EqualFold(c.in.rbeVar, "true")),
"RBE_INPUT_OFF": strconv.FormatBool(strings.EqualFold(c.in.rbeInput, "off")),
"RBE_INPUT_CACHE": strconv.FormatBool(strings.EqualFold(c.in.rbeInput, "cache")),
"PULL_REQUEST": strconv.FormatBool(c.in.pr),
"FORK": strconv.FormatBool(c.in.fork),
"DEPENDABOT": strconv.FormatBool(c.in.dependabot),
"FORK_FARM": strconv.FormatBool(c.in.farm),
"HAS_EXECUTOR": strconv.FormatBool(c.in.secret != ""),
"PR_NUMBER": "",
bazelTestMintEnv: c.in.mint,
}
if c.in.pr || c.in.farm {
env["PR_NUMBER"] = "7123"
}
out, log, err := runBazelRBEDecisionLog(t, step.Run, env)
if err != nil {
t.Fatal(err)
}
want := map[string]string{"mode": c.mode, "enabled": c.enabled, "tier": c.tier}
if !reflect.DeepEqual(out, want) {
t.Errorf("outputs = %v, want %v", out, want)
}
// The mint is asked exactly for fork and Dependabot
// pull_request runs not forced local or cache, about this run.
asked := c.in.pr && (c.in.fork || c.in.dependabot) && c.in.rbeInput == ""
if got := strings.Count(log, bazelMintStatusURL); got != map[bool]int{false: 0, true: 1}[asked] {
t.Errorf("mint status asked %d times, want %v:\n%s", got, asked, log)
}
if asked && !strings.Contains(log, bazelMintStatusURL+"beads&run=4242&attempt=2&pr=7123") {
t.Errorf("mint status URL does not name this run:\n%s", log)
}
// A closed gate drops the connection: each try gives up in 5 s.
if asked && !strings.Contains(log, "curl -sS --connect-timeout 5 --max-time 30 ") {
t.Errorf("mint status asked without --connect-timeout 5 --max-time 30:\n%s", log)
}
})
}
// A value that is not a boolean fails the job rather than picking a mode.
if out, err := runBazelRBEDecision(t, step.Run, map[string]string{
"RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "RBE_INPUT_CACHE": "false", "PULL_REQUEST": "false", "FORK": "", "DEPENDABOT": "false", "FORK_FARM": "false", "HAS_EXECUTOR": "true",
}); err == nil {
t.Errorf("decision with FORK='' succeeded with %v; want failure", out)
}
if out, err := runBazelRBEDecision(t, step.Run, map[string]string{
"RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "PULL_REQUEST": "false", "FORK": "false", "DEPENDABOT": "false", "FORK_FARM": "false", "HAS_EXECUTOR": "true",
}); err == nil {
t.Errorf("decision without RBE_INPUT_CACHE succeeded with %v; want failure", out)
}
// A fork pull_request run without a sane PR number fails rather than
// asking the mint about something else.
for _, n := range []string{"", "0", "12a", "1&repo=gascity", "1234567890"} {
if out, err := runBazelRBEDecision(t, step.Run, map[string]string{
"RBE_VAR_ON": "false", "RBE_INPUT_OFF": "false", "RBE_INPUT_CACHE": "false", "PULL_REQUEST": "true", "FORK": "true", "DEPENDABOT": "false",
"FORK_FARM": "false", "HAS_EXECUTOR": "false", "PR_NUMBER": n, bazelTestMintEnv: "ro",
}); err == nil {
t.Errorf("decision with PR_NUMBER %q succeeded with %v; want failure", n, out)
}
}
}
// bazelTestMintEnv selects runBazelRBEDecision's curl stub's answer for
// rbe-fork-mint's /v1/status: ro, rw (open, that tier), closed, rw-closed
// (open false; today's mint answers ro instead while rw is off), canary (403), busy (502), garbage, evil (open with a tier
// that is neither), or anything else (connection refused, as while the
// farm's gate is closed or DNS has no rbe-mint yet).
const bazelTestMintEnv = "BAZEL_TEST_MINT"
// bazelTestCurlStub stands in for curl on the decision step's PATH: it logs
// its arguments and prints what the mint would. Its exit status is curl's
// (no -f: HTTP errors still print the body and exit 0).
const bazelTestCurlStub = `#!/usr/bin/env bash
printf 'curl %s\n' "$*" >>"$BAZEL_TEST_CURL_LOG"
open='"instance": "oss-fork", "endpoint": "grpcs://rbe-fork.ops.gascity.com:8444"'
case "${BAZEL_TEST_MINT:-}" in
ro) echo "{\"open\": true, \"tier\": \"ro\", $open, \"reason\": \"eligible\"}" ;;
rw) echo '{"open": true, "tier": "rw", "instance": "oss", "endpoint": "grpcs://rbe-fork.ops.gascity.com:8444", "reason": "eligible"}' ;;
closed) echo "{\"open\": false, \"tier\": \"ro\", $open, \"reason\": \"rbe-fork is closed\"}" ;;
rw-closed) echo '{"open": false, "tier": "rw", "instance": "oss", "endpoint": "grpcs://rbe-fork.ops.gascity.com:8444", "reason": "the rw tier is closed"}' ;;
canary) echo '{"error": "rbe-fork canary: this PR is not enabled yet"}' ;;
busy) echo '{"error": "mint busy; retry later"}' ;;
garbage) echo '<html>bad gateway</html>' ;;
evil) echo '{"open": true, "tier": "admin", "instance": "", "endpoint": "grpcs://elsewhere:1"}' ;;
*) echo "curl: (7) Failed to connect to rbe-mint.ops.gascity.com port 8444" >&2; exit 7 ;;
esac
`
// runBazelRBEDecision runs the rbe job's step script under bash -e (as
// GitHub's default shell) and returns its $GITHUB_OUTPUT. curl is
// bazelTestCurlStub (env bazelTestMintEnv picks the mint's answer), so no
// test reaches the network.
func runBazelRBEDecision(t *testing.T, script string, env map[string]string) (map[string]string, error) {
t.Helper()
out, _, err := runBazelRBEDecisionLog(t, script, env)
return out, err
}
// runBazelRBEDecisionLog: runBazelRBEDecision, plus the stub's curl log.
func runBazelRBEDecisionLog(t *testing.T, script string, env map[string]string) (map[string]string, string, error) {
t.Helper()
dir := t.TempDir()
output := filepath.Join(dir, "output")
bin := filepath.Join(dir, "bin")
curlLog := filepath.Join(dir, "curl.log")
if err := os.MkdirAll(bin, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(bin, "curl"), []byte(bazelTestCurlStub), 0o755); err != nil {
t.Fatal(err)
}
cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script)
cmd.Env = []string{"PATH=" + bin + string(os.PathListSeparator) + os.Getenv("PATH"), "GITHUB_OUTPUT=" + output, "GITHUB_STEP_SUMMARY=" + filepath.Join(dir, "summary"),
"GITHUB_REPOSITORY=gastownhall/beads", "GITHUB_RUN_ID=4242", "GITHUB_RUN_ATTEMPT=2", "BAZEL_TEST_CURL_LOG=" + curlLog}
for k, v := range env {
cmd.Env = append(cmd.Env, k+"="+v)
}
b, err := cmd.CombinedOutput()
logData, _ := os.ReadFile(curlLog)
if err != nil {
return nil, string(logData), fmt.Errorf("%v: %s", err, b)
}
data, err := os.ReadFile(output)
if err != nil {
return nil, string(logData), err
}
out := map[string]string{}
for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
k, v, _ := strings.Cut(line, "=")
out[k] = v
}
return out, string(logData), nil
}
// write-bazelrc.sh: no secrets means a local-only rc, plus --config=fork-cache
// with BAZEL_FORK_CACHE=true (fork PRs), a partial set (or the fork cache
// with the secrets) fails, and the full set enables remote-exec with the key
// material and rc outside the workspace and the runner cache.
func TestSetupBazelRCWriter(t *testing.T) {
bash := requireHostTool(t, "bash")
script := filepath.Join(sourceRepoRoot(t), setupBazelActionDir, "write-bazelrc.sh")
pem := func(kind string) string {
body := "-----BEGIN " + kind + "-----\nMIIBfake" + strings.ReplaceAll(kind, " ", "") + "\nline2\n-----END " + kind + "-----\n"
return base64.StdEncoding.EncodeToString([]byte(body))
}
run := func(t *testing.T, extra ...string) (string, string, string, error) {
t.Helper()
dir := t.TempDir()
workspace := filepath.Join(dir, "ws")
if err := os.MkdirAll(workspace, 0o755); err != nil {
t.Fatal(err)
}
out := filepath.Join(dir, "github_output")
secret := filepath.Join(dir, "secret")
cmd := exec.Command(bash, script)
cmd.Dir = workspace
cmd.Env = append([]string{
"PATH=" + os.Getenv("PATH"),
"GITHUB_WORKSPACE=" + workspace,
"GITHUB_OUTPUT=" + out,
"BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"),
"BAZEL_CI_SECRET_DIR=" + secret,
// The fork cache's zstd probe: a refused port unless extra
// names a stand-in rbe-cache; never rbe-cache itself.
"RBE_CACHE_PROBE_URL=" + refusedProbeURL,
}, extra...)
logs, err := cmd.CombinedOutput()
outputs, _ := os.ReadFile(out)
rc, _ := os.ReadFile(filepath.Join(secret, "ci.bazelrc"))
return string(outputs), string(rc), string(logs), err
}
executor := "BAZEL_REMOTE_EXECUTOR=grpcs://" + "farm.invalid:443"
t.Run("no secrets stays local", func(t *testing.T) {
outputs, rc, logs, err := run(t)
if err != nil {
t.Fatalf("err=%v\n%s", err, logs)
}
if !strings.Contains(outputs, "remote=false") || strings.Contains(rc, "remote") || strings.Contains(rc, "tls") {
t.Errorf("outputs=%q rc=%q; want remote=false and no remote flags", outputs, rc)
}
if !strings.Contains(rc, "--repository_cache=") || strings.Contains(rc, "--disk_cache=") {
t.Errorf("local rc = %q; want --repository_cache and no --disk_cache (local runs never save it)", rc)
}
// The repo contents cache (extracted repos, never re-verified) would
// live in the runner cache: off.
if !strings.Contains(rc, "\ncommon --repo_contents_cache=\n") {
t.Errorf("rc = %q; want common --repo_contents_cache= (disabled)", rc)
}
if !strings.Contains(outputs, "cache=false") || strings.Contains(rc, "fork-cache") {
t.Errorf("outputs=%q rc=%q; want cache=false and no fork-cache line", outputs, rc)
}
})
t.Run("fork cache without secrets", func(t *testing.T) {
outputs, rc, logs, err := run(t, "BAZEL_FORK_CACHE=true")
if err != nil {
t.Fatalf("err=%v\n%s", err, logs)
}
if !strings.Contains(rc, "\nbuild --config=fork-cache\n") {
t.Errorf("rc = %q; want build --config=fork-cache", rc)
}
if strings.Contains(rc, "remote-exec") || strings.Contains(rc, "tls") || strings.Contains(rc, "--remote_") {
t.Errorf("rc = %q; the fork cache must add nothing but --config=fork-cache", rc)
}
if !strings.Contains(outputs, "remote=false") || !strings.Contains(outputs, "cache=true") {
t.Errorf("outputs = %q, want remote=false and cache=true", outputs)
}
if !strings.Contains(logs, "setup-bazel: read-only remote cache (rbe-cache); executing locally") {
t.Errorf("log lacks the read-only cache notice:\n%s", logs)
}
})
// The fork cache asks for zstd exactly while rbe-cache advertises it
// (cache-zstd-probe.sh against a stand-in; TestCacheZstdProbe runs every
// other answer): the line follows --config=fork-cache, and the lane
// never fails on the probe. Remote-exec and rbe-fork never probe.
t.Run("fork cache zstd follows rbe-cache", func(t *testing.T) {
requireCacheZstdProbeTools(t)
for answer, wantLine := range map[string]bool{"zstd": true, "identity": false, "refused": false} {
env := []string{"BAZEL_FORK_CACHE=true"}
if answer != "refused" {
body := capsZstd
if answer == "identity" {
body = capsLive
}
url, ca, _ := serveCapabilities(t, capsAnswer{grpc: "0", body: grpcMessage(body)})
env = append(env, "RBE_CACHE_PROBE_URL="+url, "CURL_CA_BUNDLE="+ca)
}
outputs, rc, logs, err := run(t, env...)
if err != nil {
t.Fatalf("%s: err=%v\n%s", answer, err, logs)
}
if got := strings.Contains(rc, "\nbuild --config=fork-cache\n"+forkCacheZstdLine+"\n"); got != wantLine || strings.Count(rc, "remote_cache_compression") != map[bool]int{true: 1}[wantLine] {
t.Errorf("rbe-cache %s: rc = %q; want %q right after --config=fork-cache: %v", answer, rc, forkCacheZstdLine, wantLine)
}
if strings.Contains(rc, "remote-exec") || !strings.Contains(outputs, "cache=true") {
t.Errorf("rbe-cache %s: outputs = %q rc = %q; want cache=true and no remote-exec", answer, outputs, rc)
}
if !strings.Contains(logs, "rbe-cache zstd probe: ") {
t.Errorf("rbe-cache %s: log lacks the probe's verdict:\n%s", answer, logs)
}
}
})
// The fork cache of a job that opts in (BAZEL_RRC_READ=true: the remote
// repo contents cache readers, bazel_rrc_test.go) reads the remote repo
// contents cache exactly while rbe-cache answers (cache-rrc-probe.sh
// against a stand-in; TestCacheRRCProbe runs every other answer and the
// kill switch): the reader's two lines, after --config=fork-cache and
// any zstd line, and nothing else; otherwise none, with a notice saying
// why. A job that does not opt in never probes. The step summary says
// which, and why. The lane never fails on the probe.
t.Run("fork cache repo contents cache follows rbe-cache", func(t *testing.T) {
requireCacheZstdProbeTools(t)
for answer, wantRead := range map[string]bool{"answers": true, "zstd": true, "denied": false, "refused": false, "not a reader": false} {
summaryFile := filepath.Join(t.TempDir(), "summary.md")
env := []string{"BAZEL_FORK_CACHE=true", "BAZEL_RRC_READ=true", "GITHUB_STEP_SUMMARY=" + summaryFile}
switch answer {
case "answers":
url, ca, _ := serveCapabilities(t, capsAnswer{grpc: "0", body: grpcMessage(capsLive)})
env = append(env, "RBE_CACHE_PROBE_URL="+url, "CURL_CA_BUNDLE="+ca)
case "not a reader":
url, ca, _ := serveCapabilities(t, capsAnswer{grpc: "0", body: grpcMessage(capsLive)})
env = []string{"BAZEL_FORK_CACHE=true", "GITHUB_STEP_SUMMARY=" + summaryFile, "RBE_CACHE_PROBE_URL=" + url, "CURL_CA_BUNDLE=" + ca}
case "zstd":
url, ca, _ := serveCapabilities(t, capsAnswer{grpc: "0", body: grpcMessage(capsZstd)})
env = append(env, "RBE_CACHE_PROBE_URL="+url, "CURL_CA_BUNDLE="+ca)
case "denied":
url, ca, _ := serveCapabilities(t, capsAnswer{grpc: "7"})
env = append(env, "RBE_CACHE_PROBE_URL="+url, "CURL_CA_BUNDLE="+ca)
}
outputs, rc, logs, err := run(t, env...)
if err != nil {
t.Fatalf("%s: err=%v\n%s", answer, err, logs)
}
_, after, ok := strings.Cut(rc, "\nbuild --config=fork-cache\n")
if !ok {
t.Fatalf("%s: rc lacks build --config=fork-cache:\n%s", answer, rc)
}
var want, got []string
if answer == "zstd" {
want = append(want, forkCacheZstdLine)
}
if wantRead {
want = append(want, bazelRRCReadLines...)
}
if after = strings.TrimSpace(after); after != "" {
got = strings.Split(after, "\n")
}
if !slices.Equal(got, want) {
t.Errorf("rbe-cache %s: rc after --config=fork-cache %q, want exactly %q", answer, got, want)
}
for _, line := range bazelRRCReadLines {
if strings.Count(rc, line) != map[bool]int{true: 1}[wantRead] {
t.Errorf("rbe-cache %s: rc has %q %d times, want %v", answer, line, strings.Count(rc, line), wantRead)
}
}
if !strings.Contains(outputs, "cache=true") || strings.Contains(rc, "remote-exec") || strings.Contains(rc, "--remote_upload_local_results") {
t.Errorf("rbe-cache %s: outputs = %q rc = %q; want cache=true, no remote-exec and no upload", answer, outputs, rc)
}
probed := answer != "not a reader"
notice := "::notice title=No remote repo contents cache::rbe-cache rrc probe: "
if got := strings.Contains(logs, notice); got != (probed && !wantRead) {
t.Errorf("rbe-cache %s: notice %v, want %v (the probe's reason, only when it fails):\n%s", answer, got, probed && !wantRead, logs)
}
if got := strings.Contains(logs, "rbe-cache rrc probe: "); got != probed {
t.Errorf("rbe-cache %s: log has the probe's verdict %v, want %v:\n%s", answer, got, probed, logs)
}
summary, _ := os.ReadFile(summaryFile)
wantSummary := map[string]string{
"answers": "**on** (rbe-cache answers GetCapabilities, GetActionResult and ByteStream Read (tcp connect ",
"zstd": "**on** (rbe-cache answers GetCapabilities, GetActionResult and ByteStream Read (tcp connect ",
"denied": "**off** (gRPC status 7 (tcp connect ",
"refused": "**off** (",
"not a reader": "**off** (not a remote repo contents cache reader (no BAZEL_RRC_READ))\n",
}[answer]
if !strings.HasPrefix(string(summary), "Fork cache remote repo contents cache read: "+wantSummary) || strings.Count(string(summary), "\n") != 1 {
t.Errorf("rbe-cache %s: step summary %q, want one line starting %q", answer, summary, wantSummary)
}
}
})
t.Run("remote-exec and rbe-fork never ask for zstd", func(t *testing.T) {
requireCacheZstdProbeTools(t)
url, ca, hits := serveCapabilities(t, capsAnswer{grpc: "0", body: grpcMessage(capsZstd)})
probe := []string{"RBE_CACHE_PROBE_URL=" + url, "CURL_CA_BUNDLE=" + ca}
for name, env := range map[string][]string{
"remote-exec": {executor, "RBE_TLS_CERT=" + pem("CERTIFICATE"), "RBE_TLS_KEY=" + pem("PRIVATE KEY")},
"rbe-fork": {"RBE_FORK_CERT_FILE=/tmp/s/fork.crt", "RBE_FORK_KEY_FILE=/tmp/s/fork.key",
"RBE_FORK_ENDPOINT=grpcs://rbe-fork.ops.gascity.com:8444", "RBE_FORK_INSTANCE=oss-fork"},
} {
_, rc, logs, err := run(t, append(env, probe...)...)
if err != nil {
t.Fatalf("%s: err=%v\n%s", name, err, logs)
}
if strings.Contains(rc, "remote_cache_compression") {
t.Errorf("%s rc asks for compression; rbe-west's schedulers and rbe-fork advertise none:\n%s", name, rc)
}
// Their repo contents cache readers are bazel.yml's own steps
// (mode remote, TestBazelRRCReadSteps); never this rc.
for _, line := range bazelRRCReadLines {
if strings.Contains(rc, line) {
t.Errorf("%s rc carries %q; only the fork cache's rc reads through rbe-cache:\n%s", name, line, rc)
}
}
}
if n := hits.Load(); n != 0 {
t.Errorf("remote-exec or rbe-fork probed rbe-cache %d times; only the fork cache may", n)
}
})
t.Run("rejects fork cache with the secrets", func(t *testing.T) {
_, rc, logs, err := run(t, "BAZEL_FORK_CACHE=true", executor, "RBE_TLS_CERT="+pem("CERTIFICATE"), "RBE_TLS_KEY="+pem("PRIVATE KEY"))
if err == nil || !strings.Contains(logs, "mutually exclusive") {
t.Errorf("err=%v, want failure naming the conflict:\n%s", err, logs)
}
if strings.Contains(rc, "remote_executor") || strings.Contains(rc, "fork-cache") {
t.Errorf("rc = %q after the conflict; want neither remote-exec nor fork-cache", rc)
}
})
for _, bad := range []string{"yes", "false", "1", "TRUE"} {
t.Run("rejects BAZEL_FORK_CACHE="+bad, func(t *testing.T) {
if _, _, logs, err := run(t, "BAZEL_FORK_CACHE="+bad); err == nil {
t.Errorf("want failure, got success:\n%s", logs)
}
})
t.Run("rejects BAZEL_RRC_READ="+bad, func(t *testing.T) {
if _, _, logs, err := run(t, "BAZEL_FORK_CACHE=true", "BAZEL_RRC_READ="+bad); err == nil || !strings.Contains(logs, "BAZEL_RRC_READ must be") {
t.Errorf("want failure naming BAZEL_RRC_READ, got %v:\n%s", err, logs)
}
})
}
for name, env := range map[string][]string{
"executor only": {executor},
"cert and key": {"RBE_TLS_CERT=" + pem("CERTIFICATE"), "RBE_TLS_KEY=" + pem("PRIVATE KEY")},
"executor no key": {executor, "RBE_TLS_CERT=" + pem("CERTIFICATE")},
"cert not pem b64": {executor, "RBE_TLS_CERT=bm90IGEgcGVt", "RBE_TLS_KEY=" + pem("PRIVATE KEY")},
} {
t.Run("rejects "+name, func(t *testing.T) {
if _, _, logs, err := run(t, env...); err == nil {
t.Errorf("want failure, got success:\n%s", logs)
}
})
}
t.Run("full set enables remote-exec", func(t *testing.T) {
outputs, rc, logs, err := run(t, executor, "RBE_TLS_CERT="+pem("CERTIFICATE"), "RBE_TLS_KEY="+pem("PRIVATE KEY"))
if err != nil {
t.Fatalf("err=%v\n%s", err, logs)
}
for _, want := range []string{
"build:remote-exec --remote_executor=grpcs://farm.invalid:443", "--tls_client_certificate=", "--tls_client_key=",
"build:remote-exec --noremote_upload_local_results", "build --config=remote-exec",
} {
if !strings.Contains(rc, want) {
t.Errorf("rc lacks %q:\n%s", want, rc)
}
}
if strings.Contains(rc, "--disk_cache") || strings.Contains(rc, "--remote_instance_name") {
t.Errorf("rc sets --disk_cache or an instance nobody asked for:\n%s", rc)
}
if !strings.Contains(outputs, "remote=true") || !strings.Contains(outputs, "cache=false") || strings.Contains(rc, "fork-cache") {
t.Errorf("outputs = %q, want remote=true, cache=false and no fork-cache in the rc", outputs)
}
for _, want := range []string{"::add-mask::farm.invalid\n", "::add-mask::MIIBfakeCERTIFICATE\n", "::add-mask::MIIBfakePRIVATEKEY\n", "::add-mask::line2\n", "::warning title=RBE_INSTANCE not set::"} {
if !strings.Contains(logs, want) {
t.Errorf("log lacks %q:\n%s", want, logs)
}
}
if strings.Contains(logs, "::add-mask::-----") {
t.Errorf("PEM armor lines are masked (they would blank every PEM header in the log):\n%s", logs)
}
})
t.Run("CA and instance", func(t *testing.T) {
_, rc, logs, err := run(t, executor, "RBE_TLS_CERT="+pem("CERTIFICATE"), "RBE_TLS_KEY="+pem("PRIVATE KEY"),
"RBE_TLS_CA="+pem("CA CERT"), "RBE_INSTANCE=beads")
if err != nil {
t.Fatalf("err=%v\n%s", err, logs)
}
for _, want := range []string{"build:remote-exec --tls_certificate=", "build:remote-exec --remote_instance_name=beads"} {
if !strings.Contains(rc, want) {
t.Errorf("rc lacks %q:\n%s", want, rc)
}
}
if !strings.Contains(logs, "::add-mask::MIIBfakeCACERT\n") || strings.Contains(logs, "RBE_INSTANCE not set") {
t.Errorf("want the CA masked and no instance warning:\n%s", logs)
}
})
t.Run("secret dir inside workspace fails", func(t *testing.T) {
dir := t.TempDir()
cmd := exec.Command(bash, script)
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_WORKSPACE=" + dir,
"BAZEL_CI_CACHE_DIR=" + filepath.Join(t.TempDir(), "c"), "BAZEL_CI_SECRET_DIR=" + filepath.Join(dir, "s")}
if out, err := cmd.CombinedOutput(); err == nil {
t.Errorf("want failure for a secret dir inside the workspace:\n%s", out)
}
})
}
const (
bazelAutofixWorkflowName = "bazel-autofix.yml"
bazelAutofixPushScript = "scripts/bazel-autofix-push.sh"
bazelSyncPatchScript = "scripts/ci/bazel-sync-patch.sh"
bazelSyncStepName = "BUILD files in sync (gazelle, go_srcs, MODULE.bazel)"
)
// bazel.yml's sync step stays red on drift but leaves the allowlisted part of
// the fix as the bazel-sync-patch artifact, from PR-head code that sees no
// secrets; bazel-autofix.yml is the only consumer.
func TestBazelSyncStepPublishesPatch(t *testing.T) {
job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName)
sync := job.step(t, bazelSyncStepName)
if sync.ID != "sync" {
t.Errorf("sync step id = %q, want sync", sync.ID)
}
patchCmd := "./" + bazelSyncPatchScript + ` "$RUNNER_TEMP/bazel-sync-patch"`
for _, required := range []string{"make bazel-sync-check || check=$?", patchCmd, "exit 1", `exit "$check"`} {
if !strings.Contains(sync.Run, required) {
t.Errorf("sync step does not contain %q:\n%s", required, sync.Run)
}
}
if i, j := strings.Index(sync.Run, patchCmd), strings.Index(sync.Run, "exit 1"); j < i {
t.Errorf("sync step must write the patch before failing")
}
for key, value := range sync.Env {
if strings.Contains(value, "secrets") {
t.Errorf("sync step env %s reads secrets; it runs PR-head code", key)
}
}
upload := job.step(t, "Upload BUILD sync patch")
if upload.If != "${{ always() && steps.sync.outcome == 'failure' && github.event_name != 'pull_request_target' }}" {
t.Errorf("patch upload if = %q; want it gated on the sync step's failure, never from %s's pull_request_target runs", upload.If, bazelFarmWorkflowName)
}
if upload.With["name"] != "bazel-sync-patch" || upload.With["path"] != "${{ runner.temp }}/bazel-sync-patch/" ||
upload.With["if-no-files-found"] != "ignore" {
t.Errorf("patch upload with = %v", upload.With)
}
if job.stepIndex(t, "Upload BUILD sync patch") != job.stepIndex(t, bazelSyncStepName)+1 {
t.Errorf("patch upload must directly follow the sync step")
}
}
// bazel-autofix.yml runs with write permissions via workflow_run, so it must
// never check out or execute PR code: the base branch's checkout, one trusted
// script, the artifact as data, no expressions in run bodies, and the push
// token visible only to the push step.
func TestBazelAutofixWorkflowSecurity(t *testing.T) {
rel := filepath.Join(".github", "workflows", bazelAutofixWorkflowName)
root := readYAMLNode(t, rel)
if got := yamlMapKeys(root, "on"); !reflect.DeepEqual(got, []string{"workflow_run"}) {
t.Errorf("triggers = %v, want exactly [workflow_run] (never pull_request_target)", got)
}
var doc struct {
On struct {
WorkflowRun struct {
Workflows []string `yaml:"workflows"`
Types []string `yaml:"types"`
} `yaml:"workflow_run"`
} `yaml:"on"`
Permissions map[string]string `yaml:"permissions"`
}
if err := root.Decode(&doc); err != nil {
t.Fatal(err)
}
// Only "PR" (pr.yml, whose bazel job calls bazel.yml) uploads
// bazel-sync-patch in a pull_request run: bazel.yml has no pull_request
// trigger of its own, so a "Bazel" run is never a PR event. Any other
// trigger would only ever be a no-op run (and, with a shared concurrency
// group, could get in a real fix's way).
if want := []string{"PR"}; !reflect.DeepEqual(doc.On.WorkflowRun.Workflows, want) {
t.Errorf("workflow_run.workflows = %v, want %v", doc.On.WorkflowRun.Workflows, want)
}
requireWorkflowProducesArtifact(t, "pr.yml", "PR", "bazel-sync-patch")
if slices.Contains(yamlMapKeys(readYAMLNode(t, filepath.Join(".github", "workflows", bazelWorkflowName)), "on"), "pull_request") {
t.Errorf("%s has its own pull_request trigger; its PR runs would upload bazel-sync-patch outside the \"PR\" run this workflow watches", bazelWorkflowName)
}
if !reflect.DeepEqual(doc.On.WorkflowRun.Types, []string{"completed"}) {
t.Errorf("workflow_run.types = %v, want [completed]", doc.On.WorkflowRun.Types)
}
wantPerms := map[string]string{"contents": "write", "pull-requests": "write", "actions": "read"}
if !reflect.DeepEqual(doc.Permissions, wantPerms) {
t.Errorf("permissions = %v, want exactly %v", doc.Permissions, wantPerms)
}
workflow := readCIWorkflow(t, bazelAutofixWorkflowName)
if len(workflow.Jobs) != 1 {
t.Fatalf("%s has %d jobs, want 1", bazelAutofixWorkflowName, len(workflow.Jobs))
}
job := workflow.job(t, "autofix")
for _, cond := range []string{"github.event.workflow_run.event == 'pull_request'", "github.event.workflow_run.conclusion == 'failure'"} {
if !strings.Contains(job.If, cond) {
t.Errorf("job if = %q lacks %q", job.If, cond)
}
}
if job.TimeoutMinutes == 0 {
t.Error("autofix job has no timeout-minutes")
}
// The App private key is a secret of this environment; its deployment-
// branch policy (configured outside this repo) is what actually
// restricts reading it to jobs running on main.
if job.Environment.Name != "autofix" {
t.Errorf("autofix job environment = %q, want %q", job.Environment.Name, "autofix")
}
var checkouts int
for _, step := range job.Steps {
if step.Uses != "" {
family, sha, _ := strings.Cut(step.Uses, "@")
switch family {
case "actions/checkout":
if sha != checkoutSHA {
t.Errorf("step %q uses %q; only actions/checkout@%s is allowed", step.Name, step.Uses, checkoutSHA)
}
if !reflect.DeepEqual(step.With, map[string]string{"persist-credentials": "false"}) {
t.Errorf("checkout has with %v; want only persist-credentials: false (base default branch, never a PR ref, no token on disk)", step.With)
}
checkouts++
case "actions/create-github-app-token":
if want := appTokenActionSHA; sha != want {
t.Errorf("step %q uses %q; want actions/create-github-app-token@%s", step.Name, step.Uses, want)
}
if step.If != "github.event.workflow_run.head_repository.full_name == github.repository" {
t.Errorf("app-token step if = %q, want the same-repo check (no token we hold can push to a fork)", step.If)
}
want := map[string]string{
"app-id": "${{ secrets.AUTOFIX_APP_ID }}",
"private-key": "${{ secrets.AUTOFIX_APP_PRIVATE_KEY }}",
"owner": "${{ github.repository_owner }}",
"repositories": "${{ github.event.repository.name }}",
"permission-contents": "write",
}
if !reflect.DeepEqual(step.With, want) {
t.Errorf("app-token step with = %v, want %v", step.With, want)
}
default:
t.Errorf("step %q uses %q; only actions/checkout@%s or actions/create-github-app-token@%s is allowed", step.Name, step.Uses, checkoutSHA, appTokenActionSHA)
}
}
// Event fields (branch names, commit messages) are attacker text:
// pass them through env, never interpolate them into a script.
if strings.Contains(step.Run, "${{") {
t.Errorf("step %q interpolates an expression into run", step.Name)
}
if regexp.MustCompile(`\bgit\s+(checkout|fetch|clone|worktree)\b|\bgh\s+pr\s+checkout\b|\bmake\b|\bgo\s+(run|build|test)\b`).MatchString(step.Run) {
t.Errorf("step %q fetches or runs code in the workflow itself:\n%s", step.Name, step.Run)
}
}
if checkouts != 1 {
t.Errorf("want exactly one checkout step, got %d", checkouts)
}
appToken := job.step(t, "Mint autofix app token")
push := job.step(t, "Push sync commit or leave apply recipe")
if job.stepIndex(t, appToken.Name) >= job.stepIndex(t, push.Name) {
t.Errorf("%q must run before %q", appToken.Name, push.Name)
}
if strings.TrimSpace(push.Run) != "./"+bazelAutofixPushScript {
t.Errorf("push step run = %q, want ./%s", push.Run, bazelAutofixPushScript)
}
download := job.step(t, "Download BUILD sync patch (if any)")
if !strings.Contains(download.Run, `select(.name == "bazel-sync-patch")`) ||
!strings.Contains(download.Run, "bazel-sync.patch bazel-sync-meta.txt -d") {
t.Errorf("download step must fetch bazel-sync-patch and extract only its two files:\n%s", download.Run)
}
for key, want := range map[string]string{
"HEAD_REPO": "${{ github.event.workflow_run.head_repository.full_name }}",
"HEAD_BRANCH": "${{ github.event.workflow_run.head_branch }}",
"HEAD_SHA": "${{ github.event.workflow_run.head_sha }}",
"GH_TOKEN": "${{ github.token }}",
"PUSH_TOKEN": "${{ steps.app-token.outputs.token || github.token }}",
"AUTOFIX_TOKEN_KIND": "${{ steps.app-token.outputs.token && 'app' || 'default' }}",
} {
if got := push.Env[key]; got != want {
t.Errorf("push step env %s = %q, want %q", key, got, want)
}
}
// PUSH_TOKEN must never be fed by anything a PR controls: only the
// app-token step's output (itself minted from environment secrets) or
// the workflow's own token.
if regexp.MustCompile(`\bgithub\.event\.(workflow_run|pull_request)\b`).MatchString(push.Env["PUSH_TOKEN"]) {
t.Errorf("push step env PUSH_TOKEN = %q reads a PR-controlled event field", push.Env["PUSH_TOKEN"])
}
// Secrets: only the app-token step's app-id/private-key, and only the
// autofix App credentials this workflow shares.
appTokenIndex := job.stepIndex(t, appToken.Name)
secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`)
allowed := map[string]bool{
fmt.Sprintf(".jobs.autofix.steps[%d].with.app-id", appTokenIndex): true,
fmt.Sprintf(".jobs.autofix.steps[%d].with.private-key", appTokenIndex): true,
}
walkYAML(root, "", func(path string, key bool, value string) {
if key || !secretRef.MatchString(value) {
return
}
if !allowed[path] {
t.Errorf("%s: %s reads secrets (%q)", bazelAutofixWorkflowName, path, value)
}
if refs := regexp.MustCompile(`secrets\.([A-Za-z0-9_]+)`).FindAllStringSubmatch(value, -1); len(refs) == 0 {
t.Errorf("%s: %s uses a non-literal secrets reference", bazelAutofixWorkflowName, path)
} else {
for _, ref := range refs {
if ref[1] != "AUTOFIX_APP_ID" && ref[1] != "AUTOFIX_APP_PRIVATE_KEY" {
t.Errorf("%s: %s reads secret %s; only AUTOFIX_APP_ID / AUTOFIX_APP_PRIVATE_KEY are shared with this workflow", bazelAutofixWorkflowName, path, ref[1])
}
}
}
})
}
// requireWorkflowProducesArtifact: the workflow a workflow_run trigger names
// runs on pull_request and uploads the artifact the autofix job consumes,
// from one of its own jobs or from a local workflow a job calls
// unconditionally (a called workflow's uploads belong to the caller's run,
// under the caller's run id).
func requireWorkflowProducesArtifact(t *testing.T, file, name, artifact string) {
t.Helper()
node := readYAMLNode(t, filepath.Join(".github", "workflows", file))
if got := yamlScalar(node, "name"); got != name {
t.Errorf("%s name = %q, want %q (the workflow_run trigger names it)", file, got, name)
}
if !slices.Contains(yamlMapKeys(node, "on"), "pull_request") {
t.Errorf("%s has no pull_request trigger; the autofix job only acts on PR runs", file)
}
uploads := func(wf ciWorkflow) bool {
for _, job := range wf.Jobs {
for _, step := range job.Steps {
if strings.HasPrefix(step.Uses, "actions/upload-artifact@") && step.With["name"] == artifact {
return true
}
}
}
return false
}
found := uploads(readCIWorkflow(t, file))
for _, job := range readCIWorkflow(t, file).Jobs {
if called, ok := strings.CutPrefix(job.Uses, "./.github/workflows/"); ok && job.If == "" && uploads(readCIWorkflow(t, called)) {
found = true
}
}
if !found {
t.Errorf("%s never uploads %s (itself or through a called workflow); the workflow_run trigger on it would be dead", file, artifact)
}
}
// Both autofix workflows push to the same PR branches. Job-level concurrency
// (skipped runs take no part) in ONE group per head branch, never cancelling:
// they queue instead of racing, and an unrelated completion cannot cancel a
// fix in flight.
func TestAutofixWorkflowsShareConcurrency(t *testing.T) {
const group = "autofix-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}"
for _, file := range []string{bazelAutofixWorkflowName, "docs-autofix.yml"} {
root := readYAMLNode(t, filepath.Join(".github", "workflows", file))
if slices.ContainsFunc(root.Content, func(n *yaml.Node) bool { return n.Value == "concurrency" }) {
t.Errorf("%s has workflow-level concurrency; it must be on the job so skipped runs take no part", file)
}
var doc struct {
Jobs map[string]struct {
Concurrency struct {
Group string `yaml:"group"`
CancelInProgress *bool `yaml:"cancel-in-progress"`
} `yaml:"concurrency"`
TimeoutMinutes int `yaml:"timeout-minutes"`
Steps []ciWorkflowStep `yaml:"steps"`
} `yaml:"jobs"`
}
if err := root.Decode(&doc); err != nil {
t.Fatal(err)
}
job, ok := doc.Jobs["autofix"]
if !ok || len(doc.Jobs) != 1 {
t.Fatalf("%s: want exactly one job, autofix", file)
}
if job.Concurrency.Group != group {
t.Errorf("%s job concurrency group = %q, want %q", file, job.Concurrency.Group, group)
}
if job.Concurrency.CancelInProgress == nil || *job.Concurrency.CancelInProgress {
t.Errorf("%s job concurrency must set cancel-in-progress: false", file)
}
// A hung run holds the shared, non-cancelling group: bound it.
if job.TimeoutMinutes != 10 {
t.Errorf("%s autofix job timeout-minutes = %d, want 10", file, job.TimeoutMinutes)
}
for _, step := range job.Steps {
if strings.HasPrefix(step.Uses, "actions/checkout@") && step.With["persist-credentials"] != "false" {
t.Errorf("%s checkout keeps the workflow token in .git/config; set persist-credentials: false", file)
}
}
}
requireWorkflowProducesArtifact(t, "pr.yml", "PR", "cli-docs-freshness-patch")
}
// The two push scripts share their security-critical functions verbatim, and
// both apply the same hardening.
func TestAutofixScriptsShareGuards(t *testing.T) {
root := sourceRepoRoot(t)
bazel := readPolicyFile(t, root, bazelAutofixPushScript)
docs := readPolicyFile(t, root, "scripts/docs-autofix-push.sh")
for _, name := range []string{"git_", "validate_patch", "check_staged", "post_or_update_comment", "head_branch_protected"} {
re := regexp.MustCompile(`(?ms)^` + regexp.QuoteMeta(name) + `\(\) \{\n.*?^\}\n`)
a, b := re.FindString(bazel), re.FindString(docs)
if a == "" || a != b {
t.Errorf("%s() differs between %s and docs-autofix-push.sh (or is missing)", name, bazelAutofixPushScript)
}
}
validator := regexp.MustCompile(`(?ms)^validate_patch\(\) \{\n.*?^\}\n`).FindString(bazel)
for _, want := range []string{
// Any rename/copy header, including legacy "rename old/new".
`rename |copy |`,
// Every index line, not only hex ones.
`grep -E '^index ' "$file" | grep -qvE '^index [0-9a-f]+\.\.[0-9a-f]+( 100644)?$'`,
`git apply --summary "$file"`,
`git apply --numstat -z "$file"`,
} {
if !strings.Contains(validator, want) {
t.Errorf("validate_patch lacks %q", want)
}
}
for script, body := range map[string]string{bazelAutofixPushScript: bazel, "scripts/docs-autofix-push.sh": docs} {
for _, want := range []string{
`select(.user.login == \"$COMMENT_AUTHOR\" and`,
`COMMENT_AUTHOR="github-actions[bot]"`,
`"--force-with-lease=refs/heads/$HEAD_BRANCH:$HEAD_SHA"`,
`gh api "repos/$BASE_REPO/branches/$enc"`,
`gh api "repos/$BASE_REPO/rules/branches/$enc"`,
`(.base.repo.full_name // "") == $base`,
"export GIT_LFS_SKIP_SMUDGE=1",
"git -c core.hooksPath=/dev/null",
// Only the branches the run needs, never a whole-repo clone.
"git_ init --quiet --bare",
"fetch --quiet --no-tags --filter=blob:none origin",
`"+refs/heads/$HEAD_BRANCH:refs/autofix/head"`,
// The circuit breaker reads the fetched commit and fails closed.
`if ! HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")" || [[ "$HEAD_SUBJECT" == "$AUTOFIX_SUBJECT"* ]]; then`,
`if length == 1 then .[0]`,
`git_ read-tree "$HEAD_SHA"`,
"apply --cached",
`check_staged "$HEAD_SHA"`,
} {
if !strings.Contains(body, want) {
t.Errorf("%s lacks %q", script, want)
}
}
// Heredocs hold the human recipe (git apply --index, git push), not
// commands this script runs.
code := regexp.MustCompile(`(?ms)<<'?EOF'?\n.*?^EOF\n`).ReplaceAllString(body, "")
if regexp.MustCompile(`\bgit_?\s+(-c\s+\S+\s+)*(checkout|switch|worktree|restore|reset|stash)\b|apply --index`).MatchString(code) {
t.Errorf("%s writes the PR tree to disk; apply to the index only", script)
}
if strings.Index(code, "if head_branch_protected;") > strings.Index(code, "push --quiet") {
t.Errorf("%s must check branch protection before pushing", script)
}
if regexp.MustCompile(`(?m)^\s*[^#\s][^#\n]*(\bclone\b|/commits/)`).MatchString(code) {
t.Errorf("%s clones the whole repository or reads commits through the API", script)
}
if strings.Contains(code, "fetch") && strings.Count(code, " fetch ") != 1 {
t.Errorf("%s: want exactly one fetch (head, plus base for attribution)", script)
}
// Every git command goes through git_ (no hooks), except the
// repository-free `git apply --summary/--numstat` of the validator.
for _, m := range regexp.MustCompile(`(?m)(?:^\s*|[;&|!(]\s*|\$\(\s*)git\s+(\S+)`).FindAllStringSubmatch(code, -1) {
if m[1] != "apply" && m[1] != "-c" {
t.Errorf("%s: git %s runs without the git_ wrapper", script, m[1])
}
}
if n, m := strings.Count(code, "git -c "), strings.Count(code, "git -c core.hooksPath=/dev/null "); n != 1 || m != 1 {
t.Errorf("%s: want exactly one raw `git -c` call, git_'s core.hooksPath=/dev/null; got %d", script, n)
}
}
}
// The producer (PR code) and the consumer (base-branch code) must agree on
// the allowlist, or every patch would be refused; the consumer's copy is the
// one that is enforced.
func TestBazelAutofixAllowlistsMatch(t *testing.T) {
re := regexp.MustCompile(`(?m)^BUILD_FILE_RE='([^']+)'$`)
caseRe := regexp.MustCompile(`(?m)^\s+(MODULE\.bazel \| MODULE\.bazel\.lock\) return 0 ;;\n\s+third_party/\*\) return 1 ;;)$`)
want := `^([A-Za-z0-9_+-][A-Za-z0-9_.+-]*/)*BUILD\.bazel$`
for _, script := range []string{bazelAutofixPushScript, bazelSyncPatchScript} {
body := readPolicyFile(t, sourceRepoRoot(t), script)
m := re.FindStringSubmatch(body)
if m == nil || m[1] != want {
t.Errorf("%s BUILD_FILE_RE = %v, want %q", script, m, want)
}
if !caseRe.MatchString(body) {
t.Errorf("%s lacks the MODULE.bazel / third_party cases of path_allowed", script)
}
}
compiled := regexp.MustCompile(want)
for path, ok := range map[string]bool{
"BUILD.bazel": true,
"cmd/bd/BUILD.bazel": true,
"internal/storage/dolt/BUILD.bazel": true,
"a_b/c-d/e.f/BUILD.bazel": true,
".github/BUILD.bazel": false,
"a/../BUILD.bazel": false,
"../BUILD.bazel": false,
"/BUILD.bazel": false,
"a//BUILD.bazel": false,
"a/.hidden/BUILD.bazel": false,
"BUILD.bazel.go": false,
"xBUILD.bazel": false,
"a/BUILD": false,
"a b/BUILD.bazel": false,
`"a\tb/BUILD.bazel"`: false,
} {
if compiled.MatchString(path) != ok {
t.Errorf("BUILD_FILE_RE matches %q = %v, want %v", path, !ok, ok)
}
}
}
func yamlScalar(node *yaml.Node, key string) string {
for i := 0; i+1 < len(node.Content); i += 2 {
if node.Content[i].Value == key {
return node.Content[i+1].Value
}
}
return ""
}
// Release builds sign and attest what they build, so they must not restore
// any Actions cache: setup-go's default cache is keyed predictably and falls
// back to the default branch's module and build caches, which are not
// re-verified (a poisoned build cache compiles straight into the binaries).
func TestReleaseWorkflowRestoresNoCache(t *testing.T) {
workflow := readCIWorkflow(t, "release.yml")
setupGo := 0
for jobName, job := range workflow.Jobs {
for _, step := range job.Steps {
if strings.HasPrefix(step.Uses, "actions/cache") || strings.Contains(step.Uses, "/cache@") {
t.Errorf("release.yml job %q step %q uses %s; release builds must not restore caches", jobName, step.Name, step.Uses)
}
if strings.HasPrefix(step.Uses, "actions/setup-go@") {
setupGo++
if step.With["cache"] != "false" {
t.Errorf("release.yml job %q step %q: setup-go must set cache: false (got %q)", jobName, step.Name, step.With["cache"])
}
}
}
}
if setupGo == 0 {
t.Fatal("release.yml has no setup-go step; update this test")
}
}
// Review F3: the gated Bazel lanes never retry a failing test. The legacy
// jobs they mirror (and, since D2 step 1, replace for the embedded tier) ran
// each test once, so a retry would hide a flaky failure no other job sees.
// Nothing may set --flaky_test_attempts (or --runs_per_test_detects_flakes,
// which reports a failed-then-passed test as FLAKY, not FAILED): not .bazelrc,
// a workflow, setup-bazel's generated rc, or a tools/bazel wrapper; and no
// BUILD file or macro may mark a target flaky = True (Bazel retries those up
// to three times by default).
func TestBazelGatedLanesNeverRetryFlakyTests(t *testing.T) {
root := sourceRepoRoot(t)
retry := regexp.MustCompile(`flaky_test_attempts|runs_per_test_detects_flakes`)
// Any flaky = other than a literal False/0 (a variable or macro
// parameter could be True). bazel-embedded also asks Bazel itself
// (TestBazelEmbeddedQueriesFlakyTargets).
flakyAttr := regexp.MustCompile(`\bflaky\s*=\s*([^,)\s]+)`)
checked := 0
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
rel, _ := filepath.Rel(root, path)
if d.IsDir() {
switch d.Name() {
case ".git", "node_modules", ".beads":
return filepath.SkipDir
}
// Not in the checkout: under Bazel, the --run_under wrapper's
// PATH directory (//tools/bazel:hermetic_bin) is in every
// test's runfiles, and a line-by-line scan of its pinned dolt
// binary took a minute on a remote worker.
if filepath.ToSlash(rel) == "tools/bazel/hermetic_bin" {
return filepath.SkipDir
}
return nil
}
if !isFileOrFileLink(path, d) {
return nil // bazel-* convenience symlinks
}
base := d.Name()
isBuild := base == "BUILD" || base == "BUILD.bazel" || strings.HasSuffix(base, ".bzl")
isRetrySurface := rel == ".bazelrc" || strings.HasPrefix(rel, ".github"+string(filepath.Separator)) ||
strings.HasPrefix(rel, filepath.Join("tools", "bazel")+string(filepath.Separator))
if !isBuild && !isRetrySurface {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
checked++
for i, line := range strings.Split(string(data), "\n") {
code := line
if j := strings.Index(code, "#"); j >= 0 && (isBuild || rel == ".bazelrc") {
code = code[:j]
}
if retry.MatchString(code) {
t.Errorf("%s:%d retries failing tests (%q); the gated lanes run each test once", rel, i+1, strings.TrimSpace(line))
}
if m := flakyAttr.FindStringSubmatch(code); isBuild && m != nil && m[1] != "False" && m[1] != "0" {
t.Errorf("%s:%d marks a target flaky (%q); Bazel would retry it", rel, i+1, strings.TrimSpace(line))
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
// Every BUILD file: a walk that sees fewer is not looking at the tree.
if checked < 100 {
t.Fatalf("checked only %d files; is the repository root right?", checked)
}
}
// Every artifact bazel.yml uploads has a fixed name within its run, and a
// job re-run ("Re-run failed jobs", the documented recovery for a Bazel lane
// that hit a remote cache eviction or a flake) is the same workflow run:
// upload-artifact v4+ refuses a second artifact of the same name there
// (409 Conflict) unless overwrite is set, which would turn a passing retry
// red. bazel-test uploads the bd package even after its tests fail.
func TestBazelWorkflowUploadsSurviveJobReruns(t *testing.T) {
uploads := 0
for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs {
for _, step := range job.Steps {
if !strings.HasPrefix(step.Uses, "actions/upload-artifact@") {
continue
}
uploads++
if step.With["overwrite"] != "true" {
t.Errorf("%s step %q uploads %q without overwrite: true; a job re-run's upload would fail with 409", name, step.Name, step.With["name"])
}
}
}
if uploads < 8 {
t.Errorf("found %d upload steps in %s, want at least 8", uploads, bazelWorkflowName)
}
}
func btoi(b bool) int {
if b {
return 1
}
return 0
}
// The cmd/bd Dolt-server tier (bazel.yml's bazel-cmd-dolt, advisory until
// pr.yml's BAZEL_CMD_DOLT_REQUIRED is "true": bazelFlagGatedLanes): the
// whole integration-tagged cmd/bd suite against a hermetic dolt sql-server,
// because every other lane, Bazel or go test, runs cmd/bd with
// BEADS_TEST_SKIP=dolt or only a manifest's tests, so its Dolt-gated tests
// ran nowhere. Coverage is exact by construction: the target's only test
// selection is go_test_pinned_shard.sh's, whose round-robin shards skip
// exactly the tests its pinned shards run (TestPinnedShardWrapperSplit,
// TestPinnedShardManifests), and check_testcases.py fails a shard that ran
// none.
func TestBazelCmdDoltJob(t *testing.T) {
const config, target = "doltserver-cmd", "bd_dolt_server_test"
workflow := readCIWorkflow(t, bazelWorkflowName)
job := workflow.job(t, bazelCmdDoltJobName)
assertBazelTierStep(t, job, bazelCmdDoltJobName, config)
test := job.step(t, "bazel test //... --config="+config)
// Mode cache runs the shards on a 4-CPU GitHub-hosted runner; with the
// read-only cache closed the cold tagged race build comes first (as in
// TestBazelIntegrationJob).
if test.TimeoutMinutes < 60 || job.TimeoutMinutes <= test.TimeoutMinutes || job.TimeoutMinutes > test.TimeoutMinutes+15 {
t.Errorf("%s timeouts: step %d, job %d; want a step of at least 60 and a job above it by at most 15", bazelCmdDoltJobName, test.TimeoutMinutes, job.TimeoutMinutes)
}
if logs := job.step(t, "Upload test logs"); logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != "bazel-cmd-dolt-testlogs" {
t.Errorf("test-log upload: if=%q name=%q", logs.If, logs.With["name"])
}
if !job.ContinueOnError {
t.Errorf("%s lacks job-level continue-on-error; while advisory its failure must not fail the call", bazelCmdDoltJobName)
}
// .bazelrc: exactly --config=integration's build flags (so the lanes
// share every compile), the lane's own tag, no dolt skip, no test
// selection, Go's timeout under rbe-west's action cap.
rcText := readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc")
rc := bazelrcLines(t)
for _, want := range []string{
"build:" + config + " --@rules_go//go/config:tags=gms_pure_go,integration",
"build:integration --@rules_go//go/config:tags=gms_pure_go,integration",
"test:" + config + " --@rules_go//go/config:race",
"test:integration --@rules_go//go/config:race",
"test:" + config + " --test_tag_filters=dolt-server-cmd",
"test:" + config + " --test_timeout=-1,-1,-1,1200",
"test:" + config + " --test_arg=-test.timeout=19m",
"test:" + config + " --test_arg=-test.parallel=4",
"test:" + config + " --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1",
"test:" + config + " --test_env=GO_TEST_WRAP_TESTV=1",
} {
if !rc[want] {
t.Errorf(".bazelrc lacks %q", want)
}
}
for _, o := range bazelrcLaneOptions(parseBazelrcOptions(rcText), config) {
arg, _ := strings.CutPrefix(o.flag, "--test_arg=")
arg = strings.TrimLeft(arg, "-")
switch {
case strings.HasPrefix(o.flag, "--test_env=BEADS_TEST_SKIP"):
t.Errorf("%s sets %s: the lane exists to run the Dolt-gated tests", o.source(), o.flag)
case strings.HasPrefix(o.flag, "--test_filter"),
strings.HasPrefix(arg, "test.run"), strings.HasPrefix(arg, "test.skip"), strings.HasPrefix(arg, "test.short"):
t.Errorf("%s sets %s: the lane runs every cmd/bd test", o.source(), o.flag)
case strings.HasPrefix(o.flag, "--@rules_go//go/config:") && o.command == "build" &&
o.flag != "--@rules_go//go/config:tags=gms_pure_go,integration":
t.Errorf("%s sets %s: its build must stay --config=integration's", o.source(), o.flag)
}
}
// The target: the dolt-server rules (local backend, fail closed, remote),
// the integration build's bd_test with no test selection, every env
// bd_test's own go_test sets, and the 4-vCPU runner shape.
build := readPolicyFile(t, sourceRepoRoot(t), "cmd/bd/BUILD.bazel")
var rule, bdTest string
for _, r := range bazelTopRules(stripStarlarkComments(build)) {
if m := bazelRuleNameRe.FindStringSubmatch(r); m != nil {
switch m[1] {
case target:
rule = r
case "bd_test":
bdTest = r
}
}
}
if rule == "" || bdTest == "" {
t.Fatalf("cmd/bd/BUILD.bazel lacks %s or bd_test", target)
}
for _, err := range checkDoltServerRules("cmd/bd", rule) {
t.Error(err)
}
for _, want := range []string{
`srcs = ["//tools/bazel:go_test_pinned_shard.sh"]`,
"args = [\n \"$(rootpath :dolt_server_pinned_shards.txt)\",\n \"$(rootpath :bd_test)\",\n ]",
`tags = ["dolt-server-cmd"]`,
`"GOMAXPROCS": "4"`,
`"BEADS_TEST_GIT_IDENTITY": "1"`,
} {
if !strings.Contains(rule, want) {
t.Errorf("%s lacks %s", target, want)
}
}
// 32 shards: 16 that run the slowest tests the manifest pins to them
// and 16 round-robin shards over the rest. Measured, a shard's fixed
// cost (TestMain, its own dolt sql-server) is a few seconds; the wall
// time is the serial sum of its tests, which round-robin alone left at
// 2-3x the median on the shard that drew the long tests. Change it
// together with bazel.yml's comment.
if m := regexp.MustCompile(`shard_count = (\d+)`).FindStringSubmatch(rule); m == nil || m[1] != "32" {
t.Errorf("%s shard_count = %v, want 32", target, m)
}
envKey := regexp.MustCompile(`"(BEADS_TEST_[A-Z_]+)": "\$\(rlocationpath [^)]+\)"`)
for _, m := range envKey.FindAllStringSubmatch(bdTest, -1) {
if !strings.Contains(rule, m[0]) {
t.Errorf("%s does not repeat bd_test's env %s", target, m[0])
}
}
}
// Exactly-once cmd/bd integration-build tests on PRs: bazel-integration runs
// //... minus //cmd/bd:bd_test only where the caller says bazel-cmd-dolt is
// required (inputs.cmd-dolt-required true, which only pr.yml passes and
// only as its committed BAZEL_CMD_DOLT_REQUIRED flag), and there
// bazel-cmd-dolt covers it: the same if (so it runs in every mode
// bazel-integration does, Dependabot and forks included), bazel-gate.sh
// accepts the two lanes' skips in exactly the same modes, pr.yml's gate then
// requires its result, and its target wraps the integration build's bd_test
// with no test selection (TestBazelCmdDoltJob). Everywhere else (bazel-farm,
// nightly, push, dispatch: bazel-cmd-dolt advisory) bd_test stays in
// bazel-integration. Rollback is committing "false" in both pr.yml places.
func TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers(t *testing.T) {
const input, excluded = "cmd-dolt-required", "-//cmd/bd:bd_test"
workflow := readCIWorkflow(t, bazelWorkflowName)
integ := workflow.job(t, bazelIntegJobName)
cmdDolt := workflow.job(t, bazelCmdDoltJobName)
// The input: a boolean defaulting to false, so a caller that does not
// pass it (and push/dispatch, where it is null) keeps bd_test.
if in, ok := readBazelWorkflowCall(t).Inputs[input]; !ok || in.Type != "boolean" || in.Default != "false" {
t.Errorf("%s workflow_call input %s = %+v (present %v), want type boolean, default false", bazelWorkflowName, input, in, ok)
}
// The step: the exclusion is exactly bd_test, and only when the input is
// true; nothing else narrows the target set.
test := integ.step(t, "bazel test //... --config=integration")
wantEnv := "${{ inputs." + input + " == true && '" + excluded + "' || '' }}"
if got := test.Env["EXCLUDE_TARGETS"]; got != wantEnv {
t.Errorf("%s EXCLUDE_TARGETS = %q, want %q", bazelIntegJobName, got, wantEnv)
}
if strings.Count(test.Run, "${EXCLUDE_TARGETS:+-- \"$EXCLUDE_TARGETS\"}") != 1 || strings.Count(test.Run, "EXCLUDE_TARGETS:+") != 1 ||
strings.Contains(test.Run, " -- ") || strings.Contains(test.Run, "-//") {
t.Errorf("%s test step must narrow //... only through ${EXCLUDE_TARGETS:+-- \"$EXCLUDE_TARGETS\"}:\n%s", bazelIntegJobName, test.Run)
}
for name, job := range workflow.Jobs {
for k, v := range job.Env {
if strings.Contains(v, "inputs."+input) {
t.Errorf("%s job env %s reads inputs.%s; only %s's test step may", name, k, input, bazelIntegJobName)
}
}
for _, step := range job.Steps {
for k, v := range step.Env {
if strings.Contains(v, "inputs."+input) && (name != bazelIntegJobName || step.Name != test.Name || k != "EXCLUDE_TARGETS") {
t.Errorf("%s step %q env %s reads inputs.%s; only %s's EXCLUDE_TARGETS may", name, step.Name, k, input, bazelIntegJobName)
}
}
if strings.Contains(step.If, "inputs."+input) || strings.Contains(step.Run, "inputs."+input) {
t.Errorf("%s step %q reads inputs.%s", name, step.Name, input)
}
}
if strings.Contains(job.If, "inputs."+input) {
t.Errorf("%s if reads inputs.%s: the input must not change which lanes run", name, input)
}
}
// Coverage where it is dropped: bazel-cmd-dolt runs in every mode
// bazel-integration does (same if, for every caller's inputs).
if integ.If != bazelIntegIf || cmdDolt.If != integ.If {
t.Errorf("%s if = %q, %s if = %q; want both %q (the cmd/bd tier must run wherever integration drops bd_test)",
bazelIntegJobName, integ.If, bazelCmdDoltJobName, cmdDolt.If, bazelIntegIf)
}
// ... and pr.yml's gate accepts their skips in the same modes only.
requireHostTool(t, "bash")
root := sourceRepoRoot(t)
for _, mode := range bazelRBEModes {
cmd := exec.Command("bash", bazelGateScript, "skips")
cmd.Dir = root
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=" + mode, "BAZEL_RBE_ENABLED=" + bazelModeEnabled(mode)}
out, err := cmd.Output()
if err != nil {
t.Fatalf("%s skips (mode %s): %v", bazelGateScript, mode, err)
}
skips := map[string]bool{}
for _, id := range strings.Fields(string(out)) {
skips[id] = true
}
if skips["BAZEL_INTEGRATION"] != skips[bazelFlagGatedLanes[bazelCmdDoltJobName].id] {
t.Errorf("mode %s: bazel-gate.sh skips BAZEL_INTEGRATION=%v but %s=%v; they must match",
mode, skips["BAZEL_INTEGRATION"], bazelFlagGatedLanes[bazelCmdDoltJobName].id, skips[bazelFlagGatedLanes[bazelCmdDoltJobName].id])
}
}
// Callers: pr.yml passes the input as exactly its flag (whose "true"
// makes the gate require BAZEL_CMD_DOLT: bazelFlagGatedLanes); no other
// caller passes it.
pr := readCIWorkflow(t, "pr.yml")
g := bazelFlagGatedLanes[bazelCmdDoltJobName]
if got, flag := pr.job(t, "bazel").With[input], pr.Env[g.flag]; got != flag || (flag != "true" && flag != "false") {
t.Errorf("pr.yml bazel job %s = %q, %s = %q; want equal, \"true\" or \"false\" (flip both together)", input, got, g.flag, flag)
}
if !strings.Contains(pr.job(t, "ci-gate").step(t, "Evaluate CI gate").Run, bazelFlagGateRun(g)) {
t.Errorf("pr.yml's gate no longer requires %s when %s is \"true\"; bd_test would then run nowhere required", g.id, g.flag)
}
entries, err := os.ReadDir(filepath.Join(root, ".github", "workflows"))
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
if e.Name() == "pr.yml" || !(strings.HasSuffix(e.Name(), ".yml") || strings.HasSuffix(e.Name(), ".yaml")) {
continue
}
for name, job := range readCIWorkflow(t, e.Name()).Jobs {
if _, ok := job.With[input]; ok && strings.HasSuffix(job.Uses, "/"+bazelWorkflowName) {
t.Errorf("%s job %s passes %s to %s; only pr.yml may (its gate is the one that requires bazel-cmd-dolt)", e.Name(), name, input, bazelWorkflowName)
}
}
}
}
func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) {
// No workflow pulls the image: every Dolt suite runs on hermetic dolt
// sql-servers (bazel.yml's dolt-server lanes) and needs no docker, since
// pr-risk.yml's container-backed legacy tiers were retired
// (ga-96smfk.22). The helper stays for local container-backed runs.
wantCalls := map[string]int{}
workflowsDir := filepath.Join(sourceRepoRoot(t), ".github", "workflows")
workflowPaths, err := filepath.Glob(filepath.Join(workflowsDir, "*.y*ml"))
if err != nil {
t.Fatal(err)
}
for _, path := range workflowPaths {
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
text := string(data)
name := filepath.Base(path)
if got, want := strings.Count(text, "run: ./scripts/ci/pull-dolt-image.sh"), wantCalls[name]; got != want {
t.Errorf("%s retry-helper calls = %d, want %d", name, got, want)
}
if strings.Contains(text, "docker pull "+doltSQLServerImage) {
t.Errorf("%s still pulls the Dolt image without retries", name)
}
}
}