beads/scripts/ci_f7c_advisory_test.go
Julian Knutsen 8761d2c40e
ci(bazel): run the conformance e2e tier under Bazel; retire conformance.yml and the docs-mintlify docsync job (#7321)
conformance.yml ran two tiers on a GitHub runner. Tier 1 (the embedded-Dolt
TestConformance) already runs as the embedded lane's
embeddeddolt_conformance_{core,audit}_test. Tier 2, the real-binary CLI
corpus (`go test -tags 'gms_pure_go e2e' ./test/conformance`), had only a
go_library under Bazel. Its two test files now build under `e2e ||
integration`, so gazelle generates //test/conformance:conformance_test
(integration-only, against the injected non-race bd_for_tests) and
--config=integration runs it remotely in bazel-integration, which pr.yml's
gate requires. PR Core's go test builds neither tag, so the --config=ci
equivalence check is unchanged; scripts/conformance.sh keeps working with
-tags e2e as the local go test entrypoint. With both tiers covered,
conformance.yml and main.yml's warm-up compiles for it are removed.

docs-mintlify.yml drops its docsync job (`go test ./test/docsync`, run by
bazel-test as //test/docsync:docsync_test on every PR) and keeps only
Mintlify's broken-link check, which needs the network (npx mint) and the
base branch's history; its path filter narrows to what that check reads.

Refs: ga-96smfk.17

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-06 22:42:25 +00:00

1536 lines
65 KiB
Go

package scripts_test
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"
"testing"
"time"
"gopkg.in/yaml.v3"
)
// F7c (spec-f7.md §2.4, §4.3): advisory workflows moved onto a same-repo-PR
// Blacksmith runner, gained a shared "upgrade-relevant code" path filter, and
// had their matrices folded (Migration Test Harness 14 -> 3, Cross-Version
// Smoke 6 -> 2). These tests pin the invariants that make those changes safe:
// the path filter is identical where it should be, no historical version or
// scenario was dropped by a fold, no advisory job can read a secret just
// because it now names a Blacksmith label, and the Blacksmith-side setup-go
// seed in main.yml actually exists for the jobs that depend on it.
// advisoryPathFilteredWorkflows are the workflows that share the
// "upgrade-relevant code" allowlist verbatim, save for each one's own
// workflow-file and script entries (spec-f7.md §2.4). conformance.yml was
// the third until its tiers moved to Bazel (embeddeddolt_conformance_*,
// //test/conformance:conformance_test).
var advisoryPathFilteredWorkflows = []string{
"migration-test.yml",
"cross-version-smoke.yml",
}
// advisoryPathFilterBase is the shared prefix of the allowlist: any non-test
// Go change or build input. It must appear, in this order, at the start of
// each of advisoryPathFilteredWorkflows' pull_request.paths list.
var advisoryPathFilterBase = []string{
"**.go",
"!**_test.go",
"go.mod",
"go.sum",
"Makefile",
".buildflags",
}
// advisoryPathFilterOwnEntries is each workflow's own file/script additions,
// appended after advisoryPathFilterBase.
var advisoryPathFilterOwnEntries = map[string][]string{
"migration-test.yml": {
".github/workflows/migration-test.yml",
"scripts/migration-test/**",
// F7c review fix (S3): the migration harness invokes this script
// directly, but it lives at scripts/ root, not under
// scripts/migration-test/, so it was not covered by the filter.
"scripts/migrate-legacy-to-current.sh",
},
"cross-version-smoke.yml": {
".github/workflows/cross-version-smoke.yml",
"scripts/upgrade-smoke-test.sh",
},
}
type pullRequestPaths struct {
On struct {
PullRequest struct {
Paths []string `yaml:"paths"`
} `yaml:"pull_request"`
} `yaml:"on"`
}
func readPullRequestPaths(t *testing.T, file string) []string {
t.Helper()
var parsed pullRequestPaths
text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+file)
if err := yaml.Unmarshal([]byte(text), &parsed); err != nil {
t.Fatalf("parse %s: %v", file, err)
}
return parsed.On.PullRequest.Paths
}
type pushPaths struct {
On struct {
Push struct {
Paths []string `yaml:"paths"`
} `yaml:"push"`
} `yaml:"on"`
}
func readPushPaths(t *testing.T, file string) []string {
t.Helper()
var parsed pushPaths
text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+file)
if err := yaml.Unmarshal([]byte(text), &parsed); err != nil {
t.Fatalf("parse %s: %v", file, err)
}
return parsed.On.Push.Paths
}
// TestAdvisoryWorkflowPathFiltersAreIdentical pins that the shared base of
// the "upgrade-relevant code" allowlist is byte-for-byte identical, in the
// same order, across all three workflows it applies to. GitHub Actions has no
// cross-file include for `on:` triggers, so this is the fallback the spec
// explicitly allows: identical literal lists plus a policy test asserting
// they stay identical (spec-f7.md §2.4).
func TestAdvisoryWorkflowPathFiltersAreIdentical(t *testing.T) {
for _, file := range advisoryPathFilteredWorkflows {
paths := readPullRequestPaths(t, file)
if len(paths) < len(advisoryPathFilterBase) {
t.Fatalf("%s pull_request.paths = %v, too short to hold the shared base %v", file, paths, advisoryPathFilterBase)
}
got := paths[:len(advisoryPathFilterBase)]
for i, want := range advisoryPathFilterBase {
if got[i] != want {
t.Errorf("%s pull_request.paths[%d] = %q, want %q (shared base must match byte-for-byte and in order)", file, i, got[i], want)
}
}
}
}
// TestAdvisoryWorkflowPathFiltersCoverOwnInputs pins that each workflow also
// allowlists its own workflow file and the scripts/fixtures it actually
// exercises, so an edit to e.g. scripts/migration-test/** is never silently
// skipped by the filter that was added to cut unrelated-PR load.
func TestAdvisoryWorkflowPathFiltersCoverOwnInputs(t *testing.T) {
for file, want := range advisoryPathFilterOwnEntries {
paths := readPullRequestPaths(t, file)
for _, entry := range want {
if !contains(paths, entry) {
t.Errorf("%s pull_request.paths %v does not contain its own entry %q", file, paths, entry)
}
}
}
}
// TestNixBuildDropsPullRequestTriggerNotPushOrDispatch pins the one
// "delete the pull_request trigger" trigger change in F7c: nix-build.yml's
// PR coverage is fully redundant with PR Risk's required test-nix job (which
// runs `nix run .#default` plus `nix flake check -L` on every PR, a superset
// of `nix build .#default`), but push and workflow_dispatch must survive so
// the plain `nix build` path stays covered post-merge.
func TestNixBuildDropsPullRequestTriggerNotPushOrDispatch(t *testing.T) {
type nixTriggers struct {
On map[string]any `yaml:"on"`
}
var parsed nixTriggers
text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/nix-build.yml")
if err := yaml.Unmarshal([]byte(text), &parsed); err != nil {
t.Fatal(err)
}
if _, ok := parsed.On["pull_request"]; ok {
t.Errorf("nix-build.yml still has a pull_request trigger; PR Risk's test-nix job is a superset (spec-f7.md §2.4)")
}
push, ok := parsed.On["push"].(map[string]any)
if !ok {
t.Fatalf("nix-build.yml has no push trigger: %+v", parsed.On)
}
branches, _ := push["branches"].([]any)
var branchNames []string
for _, b := range branches {
branchNames = append(branchNames, fmt.Sprint(b))
}
if !contains(branchNames, "main") {
t.Errorf("nix-build.yml must still push on main: %+v", push)
}
if _, ok := parsed.On["workflow_dispatch"]; !ok {
t.Errorf("nix-build.yml must still support workflow_dispatch")
}
// PR Risk's test-nix must actually be the superset this removal leans on:
// it has to build the default package AND run the flake checks, not just
// one of the two.
prRisk := readCIWorkflow(t, "pr-risk.yml")
testNix := prRisk.job(t, "test-nix")
var sawBuild, sawFlakeCheck bool
for _, step := range testNix.Steps {
if strings.Contains(step.Run, "nix run .#default") {
sawBuild = true
}
if strings.Contains(step.Run, "nix flake check") {
sawFlakeCheck = true
}
}
if !sawBuild {
t.Error("pr-risk.yml's test-nix no longer builds/runs the default package; nix-build.yml's pull_request trigger would need to come back")
}
if !sawFlakeCheck {
t.Error("pr-risk.yml's test-nix no longer runs the flake checks")
}
}
// --- Migration Test Harness: 14 -> 3 shards, no version dropped -----------
// migrationHarnessOriginalVersions is the pre-F7c 14-leg matrix's version
// list, captured verbatim so a shard rebalance can be checked against it
// without re-deriving it from the (now folded) workflow file.
var migrationHarnessOriginalVersions = []string{
"v0.9.1", "v0.17.0", "v0.49.6", "v0.50.3", "v0.55.4", "v0.56.1",
"v0.57.0", "v0.62.0", "v0.63.3", "v1.0.0", "v1.0.1", "v1.1.0",
"v1.1.2", "v1.2.2",
}
// migrationHarnessDoltRuntimeVersions is the old per-version
// `contains(fromJSON('[...]'), matrix.version)` list that gated the "Install
// Dolt test runtime" step. It must become exactly the dolt-runtime shard.
var migrationHarnessDoltRuntimeVersions = []string{
"v0.55.4", "v0.56.1", "v0.57.0", "v0.62.0", "v1.0.1", "v1.1.0", "v1.1.2", "v1.2.2",
}
func migrationHarnessShards(t *testing.T) map[string][]string {
t.Helper()
workflow := readCIWorkflow(t, "migration-test.yml")
job := workflow.job(t, "historical-upgrades")
shards := map[string][]string{}
for _, leg := range job.Strategy.Matrix.Include {
shardAny, ok := leg.Extra["shard"]
if !ok {
t.Fatalf("migration-test.yml matrix leg %+v has no shard field", leg.Extra)
}
shard, ok := shardAny.(string)
if !ok {
t.Fatalf("migration-test.yml matrix leg shard = %#v, not a string", shardAny)
}
versionsAny, ok := leg.Extra["versions"]
if !ok {
t.Fatalf("migration-test.yml shard %q has no versions field", shard)
}
versionsJSON, ok := versionsAny.(string)
if !ok {
t.Fatalf("migration-test.yml shard %q versions = %#v, not a string", shard, versionsAny)
}
var versions []string
if err := json.Unmarshal([]byte(versionsJSON), &versions); err != nil {
t.Fatalf("migration-test.yml shard %q versions %q does not parse as a JSON string array: %v", shard, versionsJSON, err)
}
if _, dup := shards[shard]; dup {
t.Fatalf("migration-test.yml declares shard %q more than once", shard)
}
shards[shard] = versions
}
return shards
}
// TestMigrationHarnessShardsCoverAllHistoricalVersions is the equivalence
// check the fold requires: the union of the 3 shards' version lists must be
// exactly the old 14-version set, with no duplicates and nothing dropped.
func TestMigrationHarnessShardsCoverAllHistoricalVersions(t *testing.T) {
shards := migrationHarnessShards(t)
wantShardNames := []string{"src", "pre-dolt", "dolt-runtime"}
for _, name := range wantShardNames {
if _, ok := shards[name]; !ok {
t.Errorf("migration-test.yml is missing shard %q", name)
}
}
if len(shards) != len(wantShardNames) {
t.Errorf("migration-test.yml has shards %v, want exactly %v", mapKeys(shards), wantShardNames)
}
seen := map[string]string{} // version -> owning shard
var union []string
for shard, versions := range shards {
for _, v := range versions {
if owner, dup := seen[v]; dup {
t.Errorf("version %s is claimed by both shard %q and shard %q", v, owner, shard)
continue
}
seen[v] = shard
union = append(union, v)
}
}
sort.Strings(union)
want := append([]string(nil), migrationHarnessOriginalVersions...)
sort.Strings(want)
if !equalStrings(union, want) {
t.Fatalf("shard union = %v, want exactly the old 14-version set %v", union, want)
}
if !equalStrings(sortedCopy(shards["dolt-runtime"]), sortedCopy(migrationHarnessDoltRuntimeVersions)) {
t.Errorf("dolt-runtime shard = %v, want exactly the old Dolt-runtime contains() list %v", shards["dolt-runtime"], migrationHarnessDoltRuntimeVersions)
}
if !equalStrings(shards["src"], []string{"v0.9.1"}) {
t.Errorf("src shard = %v, want exactly [v0.9.1]", shards["src"])
}
}
// TestMigrationHarnessLoopExecutesBehaviorally runs the actual "Verify
// explicit historical upgrades" bash against a stub run.sh, proving the
// loop's real behavior instead of its source text (F7c review fix S1/S4/S5,
// closes mutations M1 `exit 0`, M2 `break`, M14 `| head -1`, and pins the
// fail-closed and per-version-timeout fixes the reviewer required):
// - every version in the shard is attempted even after an earlier one
// fails (a short-circuiting `exit 0`/`break` would hide the rest);
// - the step's own exit code reflects any failure;
// - failures are both ::error:: annotated and written to
// $GITHUB_STEP_SUMMARY;
// - an empty or unparsable SHARD_VERSIONS fails closed instead of
// reporting a false pass, without ever invoking run.sh;
// - a hung version is bounded by `timeout`, and the loop still reaches the
// version queued after it.
//
// A bare `./scripts/migration-test/run.sh --version "$HISTORICAL_VERSION"`
// (the pre-fold single-version invocation) must also be gone: if it came
// back, the shard would silently only test one version again.
func TestMigrationHarnessLoopExecutesBehaviorally(t *testing.T) {
requireHostTool(t, "bash")
requireHostTool(t, "jq")
requireHostTool(t, "timeout")
job := readCIWorkflow(t, "migration-test.yml").job(t, "historical-upgrades")
step := job.step(t, "Verify explicit historical upgrades")
if strings.Contains(step.Run, "$HISTORICAL_VERSION") {
t.Errorf("migration-test.yml's Verify step still references the old single-version $HISTORICAL_VERSION env var")
}
writeHarness := func(t *testing.T, runSh string) (dir, summaryFile string) {
t.Helper()
dir = t.TempDir()
scriptDir := filepath.Join(dir, "scripts", "migration-test")
if err := os.MkdirAll(scriptDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(scriptDir, "run.sh"), []byte(runSh), 0o755); err != nil {
t.Fatal(err)
}
summaryFile = filepath.Join(dir, "summary.md")
if err := os.WriteFile(summaryFile, nil, 0o644); err != nil {
t.Fatal(err)
}
return dir, summaryFile
}
baseEnv := func(shardVersions, summaryFile string) []string {
return append(os.Environ(),
"CANDIDATE_BIN=./bd",
"GIT_CONFIG_NOSYSTEM=1",
"DOLT_BIN=/nonexistent/dolt",
"SHARD_VERSIONS="+shardVersions,
"SHARD=test-shard",
"GITHUB_STEP_SUMMARY="+summaryFile,
)
}
run := func(t *testing.T, script, shardVersions, runSh string) (exitCode int, out, summary string) {
t.Helper()
dir, summaryFile := writeHarness(t, runSh)
cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script)
cmd.Dir = dir
cmd.Env = baseEnv(shardVersions, summaryFile)
outBytes, err := cmd.CombinedOutput()
code := 0
if err != nil {
exitErr, ok := err.(*exec.ExitError)
if !ok {
t.Fatalf("run loop: %v\noutput:\n%s", err, outBytes)
}
code = exitErr.ExitCode()
}
summaryBytes, readErr := os.ReadFile(summaryFile)
if readErr != nil {
t.Fatal(readErr)
}
return code, string(outBytes), string(summaryBytes)
}
t.Run("every version runs even after an early failure, and the step fails", func(t *testing.T) {
const stub = `#!/usr/bin/env bash
set -euo pipefail
v=""
while [ $# -gt 0 ]; do
if [ "$1" = "--version" ]; then v="$2"; fi
shift
done
echo "stub-ran:$v"
[ "$v" = "v2" ] && exit 1
exit 0
`
code, out, summary := run(t, step.Run, `["v1","v2","v3"]`, stub)
for _, v := range []string{"v1", "v2", "v3"} {
if !strings.Contains(out, "stub-ran:"+v) {
t.Errorf("version %s never ran; output:\n%s", v, out)
}
}
if code != 1 {
t.Errorf("exit code = %d, want 1 (v2 failed)", code)
}
if !strings.Contains(out, "::error::historical upgrade failed for v2") {
t.Errorf("missing ::error:: for v2; output:\n%s", out)
}
if strings.Contains(out, "::error::historical upgrade failed for v1") || strings.Contains(out, "::error::historical upgrade failed for v3") {
t.Errorf("v1/v3 must not be reported as failed; output:\n%s", out)
}
if !strings.Contains(summary, "v2") {
t.Errorf("GITHUB_STEP_SUMMARY does not mention the failed version v2:\n%s", summary)
}
if strings.Contains(summary, "v1") || strings.Contains(summary, "v3") {
t.Errorf("GITHUB_STEP_SUMMARY must only list failed versions:\n%s", summary)
}
})
t.Run("all versions pass, step exits 0", func(t *testing.T) {
const stub = "#!/usr/bin/env bash\nexit 0\n"
code, out, summary := run(t, step.Run, `["v1","v2"]`, stub)
if code != 0 {
t.Errorf("exit code = %d, want 0, output:\n%s", code, out)
}
if strings.TrimSpace(summary) != "" {
t.Errorf("GITHUB_STEP_SUMMARY should be untouched on an all-pass run, got:\n%s", summary)
}
})
for _, badInput := range []string{`[]`, `null`, `not-json`, `"a string, not an array"`} {
t.Run("fails closed on "+badInput, func(t *testing.T) {
const stub = "#!/usr/bin/env bash\necho ran >&2\nexit 1\n"
code, out, _ := run(t, step.Run, badInput, stub)
if code != 1 {
t.Errorf("SHARD_VERSIONS=%s: exit code = %d, want 1 (fail closed)", badInput, code)
}
if !strings.Contains(out, "::error::shard test-shard resolved to zero versions") {
t.Errorf("SHARD_VERSIONS=%s: missing the fail-closed ::error::; output:\n%s", badInput, out)
}
if strings.Contains(out, "ran") {
t.Errorf("SHARD_VERSIONS=%s: run.sh must never be invoked when the version list fails closed; output:\n%s", badInput, out)
}
})
}
t.Run("a hung version is bounded by timeout and does not hide the next version", func(t *testing.T) {
// Exercise the REAL timeout/kill-after wiring, just at durations a
// unit test can afford: substitute the pinned 8m/30s for 2s/1s. If a
// mutation removed the `timeout` wrapper entirely, this substitution
// is a no-op and the stub's 10s sleep would make this subtest's own
// assertions fail well before a developer-visible hang, bounded by
// the 20s context below.
fastStep := strings.NewReplacer("8m", "2s", "30s", "1s").Replace(step.Run)
const stub = `#!/usr/bin/env bash
v=""
while [ $# -gt 0 ]; do
if [ "$1" = "--version" ]; then v="$2"; fi
shift
done
if [ "$v" = "hangs" ]; then
sleep 10
exit 0
fi
echo "stub-ran:$v"
exit 0
`
dir, summaryFile := writeHarness(t, stub)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", fastStep)
cmd.Dir = dir
cmd.Env = baseEnv(`["hangs","after"]`, summaryFile)
start := time.Now()
out, err := cmd.CombinedOutput()
elapsed := time.Since(start)
if ctx.Err() == context.DeadlineExceeded {
t.Fatalf("loop did not return within 20s; the timeout wrapper did not bound the hang. output:\n%s", out)
}
if elapsed > 8*time.Second {
t.Errorf("loop took %s to process a version meant to be killed after ~3s; output:\n%s", elapsed, out)
}
if _, ok := err.(*exec.ExitError); !ok && err != nil {
t.Fatalf("run loop: %v\noutput:\n%s", err, out)
}
if !strings.Contains(string(out), "stub-ran:after") {
t.Errorf("the version after the hang never ran; output:\n%s", out)
}
if !strings.Contains(string(out), "::error::historical upgrade failed for hangs") {
t.Errorf("the killed version was not reported as failed; output:\n%s", out)
}
})
}
// TestMigrationHarnessDoltRuntimeStepGatedByShard pins that the "Install Dolt
// test runtime" step's gate is exactly `matrix.shard == 'dolt-runtime'`, not
// some other formulation that might accidentally also match (or exclude) a
// different shard (F7c review fix S1, closes mutation M3).
func TestMigrationHarnessDoltRuntimeStepGatedByShard(t *testing.T) {
job := readCIWorkflow(t, "migration-test.yml").job(t, "historical-upgrades")
step := job.step(t, "Install Dolt test runtime")
const want = "matrix.shard == 'dolt-runtime'"
if step.If != want {
t.Errorf("migration-test.yml's Install Dolt test runtime if = %q, want %q", step.If, want)
}
}
// TestMigrationHarnessCacheHashMatchesDerivation pins that each shard's
// cache-hash is still sha256(join(",", <shard's per-version cache-identity
// strings>))[:16], computed from the pre-fold 14-entry identity table that
// existed before the 14 -> 3 shard fold (git show
// 362eecc6bf:.github/workflows/migration-test.yml), so a future shard
// rebalance or version-list edit can't silently leave a stale cache key
// behind (F7c review fix S1, closes mutation M5).
func TestMigrationHarnessCacheHashMatchesDerivation(t *testing.T) {
identities := map[string]string{
"v0.9.1": "v0.9.1-source-e3c8554fa2c3e4b9caf7e296e9c8abbe24211a72-go1.25.0-cgo1",
"v0.17.0": "v0.17.0-d4d08617a324c85b45c9628bc519d659a9ff9c7c37da67aa48727e0af7f19a75",
"v0.49.6": "v0.49.6-8546dc9a47e11dc31ac2bc9a0224a9c690975e91850932cbb62623053fbb7db8",
"v0.50.3": "v0.50.3-e94b09e0b6a9324bbc0e81ea36bccaaa42172a926bfedfb389e9a26dedb63184",
"v0.55.4": "v0.55.4-e0fa25456dd82890230eef17653448a0bf995104c78864be91c5ed84426a5f49",
"v0.56.1": "v0.56.1-4f9f6cc44465a11613ff529009901eaaf841c6b1f91c15e002b0ecda2015a15c",
"v0.57.0": "v0.57.0-f8629d5627bed7d25f06f92334addc171d679f9aed9d08c5d42a9684205dc04b",
"v0.62.0": "v0.62.0-4cca7265b22e5c3ca8d62ab0b9752bec31f68b7f5fa636282a4c7e5454c35535",
"v0.63.3": "v0.63.3-5f4efd2e010209b3f381dbcd783b2a3a652f50ea72f40ef04c8ba434d408bf9e",
"v1.0.0": "v1.0.0-7057db1e92428fcf5c08d5dc6b07ead57e588b262cba78b9a26893d55bd29fdb",
"v1.0.1": "v1.0.1-1d2364d5d7083a4634a9e734ca87822fb79c2b6625988f9f791e3376313b1b77",
"v1.1.0": "v1.1.0-b0f3dd607c3fb989ee08d0a6854fba80d0402971eb108f9af6170bc14d491a34",
"v1.1.2": "v1.1.2-a72d71ed374955dc9f83a0f90b54bd7b6a0016709dd1676ae2e368651ed401c2",
"v1.2.2": "v1.2.2-8140098a51d3b81d5548d1c5e6db1a2d9930e5d141efe2a4bff7d079c4d321e8",
}
job := readCIWorkflow(t, "migration-test.yml").job(t, "historical-upgrades")
for _, leg := range job.Strategy.Matrix.Include {
shard, _ := leg.Extra["shard"].(string)
versionsJSON, _ := leg.Extra["versions"].(string)
var versions []string
if err := json.Unmarshal([]byte(versionsJSON), &versions); err != nil {
t.Fatalf("shard %q versions %q: %v", shard, versionsJSON, err)
}
var parts []string
for _, v := range versions {
id, ok := identities[v]
if !ok {
t.Fatalf("shard %q version %q has no known pre-fold cache-identity; update the table in this test", shard, v)
}
parts = append(parts, id)
}
sum := sha256.Sum256([]byte(strings.Join(parts, ",")))
want := hex.EncodeToString(sum[:])[:16]
gotAny, ok := leg.Extra["cache-hash"]
if !ok {
t.Fatalf("shard %q has no cache-hash field", shard)
}
if got := fmt.Sprint(gotAny); got != want {
t.Errorf("shard %q cache-hash = %q, want sha256(join(\",\", identities))[:16] = %q", shard, got, want)
}
}
}
func mapKeys[K comparable, V any](m map[K]V) []K {
keys := make([]K, 0, len(m))
for k := range m {
keys = append(keys, k)
}
return keys
}
func sortedCopy(items []string) []string {
out := append([]string(nil), items...)
sort.Strings(out)
return out
}
// --- Cross-Version Smoke: 6 -> 2 jobs, chunks of 5, no version dropped ----
// ghExtractJQProgram pulls the single-quoted jq program out of a standalone
// `jq -<flags> '<program>'` invocation inside a step's `run:` text, so a test
// can execute the REAL program with `jq` directly instead of asserting on a
// substring of the surrounding bash. The flag group is required (one or
// more) specifically so this does not also match `gh`'s own `--jq
// '[.[].tagName]'` filter, which has no `-c`/`-r` flag of its own between
// "jq" and the quoted program. None of this repo's embedded jq programs
// contain a literal single quote, so a non-greedy single-quote match is
// sufficient.
var jqProgramPattern = regexp.MustCompile(`jq(?: -[A-Za-z]+)+ '([^']*)'`)
func ghExtractJQProgram(t *testing.T, run string) string {
t.Helper()
m := jqProgramPattern.FindStringSubmatch(run)
if m == nil {
t.Fatalf("no `jq '...'` invocation found in:\n%s", run)
}
return m[1]
}
func runJQ(t *testing.T, program string, stdin string) string {
t.Helper()
requireHostTool(t, "jq")
cmd := exec.Command("jq", "-c", program)
cmd.Stdin = strings.NewReader(stdin)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("jq -c %q <<<%q: %v\n%s", program, stdin, err, out)
}
return strings.TrimSpace(string(out))
}
// TestCrossVersionSmokeChunksEveryResolvedVersion runs the REAL jq programs
// embedded in the "Resolve release versions" and "Compute chunk cache key"
// steps against synthetic version lists, instead of asserting on their
// source text (F7c review fix S1, closes mutation M6 `[range(0; length; 5)
// as $i | .[$i:$i+4]]` off-by-one and M7 jq-slice-dropping-versions
// mutations): every resolved version must appear in exactly one chunk, no
// chunk may exceed 5 versions, and the chunk-key step's join must reproduce
// every version in its chunk, in order, space-separated.
func TestCrossVersionSmokeChunksEveryResolvedVersion(t *testing.T) {
workflow := readCIWorkflow(t, "cross-version-smoke.yml")
versionsJob := workflow.job(t, "versions")
resolve := versionsJob.step(t, "Resolve release versions")
chunkProgram := ghExtractJQProgram(t, resolve.Run)
if versionsJob.Outputs["chunks"] == "" {
t.Errorf("cross-version-smoke.yml's versions job has no chunks output")
}
for _, n := range []int{0, 1, 4, 5, 6, 9, 10, 29, 30} {
versions := make([]string, 0, n)
for i := 0; i < n; i++ {
versions = append(versions, fmt.Sprintf("v0.%d.0", i))
}
versionsJSON, err := json.Marshal(versions)
if err != nil {
t.Fatal(err)
}
var chunks [][]string
if err := json.Unmarshal([]byte(runJQ(t, chunkProgram, string(versionsJSON))), &chunks); err != nil {
t.Fatalf("n=%d: chunk output did not parse as [][]string: %v", n, err)
}
wantChunks := (n + 4) / 5
if n == 0 {
wantChunks = 0
}
if len(chunks) != wantChunks {
t.Errorf("n=%d: got %d chunks, want %d", n, len(chunks), wantChunks)
}
var flat []string
for _, c := range chunks {
if len(c) > 5 {
t.Errorf("n=%d: chunk %v has more than 5 versions", n, c)
}
flat = append(flat, c...)
}
if !equalStrings(flat, versions) {
t.Errorf("n=%d: concatenated chunks = %v, want exactly the resolved version list %v in order", n, flat, versions)
}
}
smokeJob := workflow.job(t, "smoke")
if !contains(smokeJob.Needs, "versions") {
t.Errorf("cross-version-smoke.yml's smoke job does not need versions")
}
chunkKeyStep := smokeJob.step(t, "Compute chunk cache key")
joinProgram := ghExtractJQProgram(t, chunkKeyStep.Run)
for _, chunk := range [][]string{
{"v1.2.2"},
{"v1.0.0", "v1.0.1", "v1.1.0", "v1.1.2", "v1.2.2"},
} {
chunkJSON, err := json.Marshal(chunk)
if err != nil {
t.Fatal(err)
}
got := strings.Trim(runJQ(t, joinProgram, string(chunkJSON)), `"`)
want := strings.Join(chunk, " ")
if got != want {
t.Errorf("chunk-key join(%v) = %q, want %q", chunk, got, want)
}
}
buildStep := smokeJob.stepIndex(t, "Build candidate binary")
runStep := smokeJob.stepIndex(t, "Run upgrade smoke tests")
if buildStep >= runStep {
t.Errorf("Build candidate binary (index %d) must run before Run upgrade smoke tests (index %d), so the candidate is built once per chunk, not once per version", buildStep, runStep)
}
run := smokeJob.step(t, "Run upgrade smoke tests")
if run.Env["SMOKE_VERSIONS"] != "${{ steps.chunk.outputs.versions }}" {
t.Errorf("cross-version-smoke.yml's smoke job SMOKE_VERSIONS = %q, want the chunk step's versions output", run.Env["SMOKE_VERSIONS"])
}
// Exact match (not a substring check, F7c review fix S1): the whole point
// of SMOKE_VERSIONS is that upgrade-smoke-test.sh's own loop consumes it;
// any extra positional arg (e.g. a reintroduced matrix.prev_version) would
// silently make the script test only one version per chunk again.
if want := "./scripts/upgrade-smoke-test.sh"; strings.TrimSpace(run.Run) != want {
t.Errorf("cross-version-smoke.yml's Run upgrade smoke tests run = %q, want exactly %q", run.Run, want)
}
}
// --- Security: no job gains a secret just by naming a Blacksmith label ----
// blacksmithAdvisoryWorkflows are every workflow file F7c moved a job onto a
// same-repo-PR (or push-only, for main.yml's seed) Blacksmith runner.
var blacksmithAdvisoryWorkflows = []string{
"regression.yml",
"migration-test.yml",
"cross-version-smoke.yml",
"docs-mintlify.yml",
"proxied-local-smoke.yml",
"main.yml",
}
// TestBlacksmithAdvisoryJobsReadNoSecrets is the security invariant spec-f7.md
// §3 requires: "a new policy test asserts that no job whose runs-on names
// blacksmith- reads secrets.". pr-risk.yml's own no-secrets walk is separate
// and untouched; this one covers the F7c advisory workflows plus main.yml's
// new seed job.
func TestBlacksmithAdvisoryJobsReadNoSecrets(t *testing.T) {
for _, file := range blacksmithAdvisoryWorkflows {
workflow := readCIWorkflow(t, file)
for name, job := range workflow.Jobs {
if !strings.Contains(job.RunsOn, "blacksmith-") {
continue
}
t.Run(file+"/"+name, func(t *testing.T) {
if containsSecretRef(fmt.Sprint(job.Env)) {
t.Errorf("%s job %s's env references secrets.", file, name)
}
for _, step := range job.Steps {
if containsSecretRef(step.Run) {
t.Errorf("%s job %s step %q's run references secrets.", file, name, step.Name)
}
if containsSecretRef(fmt.Sprint(step.Env)) {
t.Errorf("%s job %s step %q's env references secrets.", file, name, step.Name)
}
if containsSecretRef(fmt.Sprint(step.With)) {
t.Errorf("%s job %s step %q's with references secrets.", file, name, step.Name)
}
}
})
}
}
}
func containsSecretRef(s string) bool {
return strings.Contains(s, "secrets.")
}
// advisoryBlacksmithRunnerJobs pins which F7c advisory job runs on which
// same-repo Blacksmith expression, and what label each falls back to when
// the same-repo/merge_group condition is false. migration-test.yml's
// historical-upgrades job is the one case with a non-ubuntu-latest fallback
// (sameRepoBlacksmith4vcpuNoble, F7c review fix B1): it must still resolve to
// the literal ubuntu-24.04 label, not the usual ubuntu-latest, for forks/
// Dependabot/push.
var advisoryBlacksmithRunnerJobs = []struct {
file string
job string
blacksmithLabel string
fallback string
}{
{"regression.yml", "regression", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"},
{"migration-test.yml", "historical-upgrades", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-24.04"},
{"cross-version-smoke.yml", "smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"},
{"cross-version-smoke.yml", "versions", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"},
{"docs-mintlify.yml", "broken-links", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"},
{"proxied-local-smoke.yml", "managed-local-smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"},
}
// TestF7cAdvisorySameRepoBlacksmithExpressionSemantics evaluates each F7c
// advisory job's REAL, as-parsed-from-YAML runs-on expression text through
// F7a's shared evalGHExpr/mustEvalGHRunsOn (ci_blacksmith_runner_test.go),
// against a truth table covering every event shape the advisory workflows
// see: merge_group, a same-repo PR, a fork PR, a Dependabot PR, a deleted
// fork head, and push/pull_request_target/workflow_dispatch/schedule.
//
// This replaces an earlier, F7c-local hand-written Go mirror of the same
// boolean logic (a tautology risk the F7a review flagged for its own
// equivalent test: a change to both the mirror and the real expression, in
// the same wrong way, would still pass). Per the coordinator, this test now
// reuses F7a's real evaluator instead of building a second one, and is
// scoped to the F7c advisory jobs specifically; F7a's own
// TestSameRepoBlacksmithExpressionSemantics (ci_blacksmith_runner_test.go)
// covers the shared sameRepoBlacksmith{2,4,8}vcpu consts themselves.
func TestF7cAdvisorySameRepoBlacksmithExpressionSemantics(t *testing.T) {
const ownRepo = "steveyegge/beads"
type tc struct {
name string
event string
headRepo string // github.event.pull_request.head.repo.full_name; "" = fork or deleted fork head
actor string
wantBlacksmith bool
}
cases := []tc{
{"same-repo PR, human actor", "pull_request", ownRepo, "alice", true},
{"merge_group always Blacksmith", "merge_group", "", "", true},
{"fork PR stays on fallback", "pull_request", "someone-else/beads", "alice", false},
{"deleted fork head stays on fallback", "pull_request", "", "alice", false},
{"same-repo PR, dependabot actor stays on fallback", "pull_request", ownRepo, "dependabot[bot]", false},
{"push stays on fallback", "push", "", "alice", false},
{"pull_request_target stays on fallback", "pull_request_target", ownRepo, "alice", false},
{"schedule stays on fallback", "schedule", "", "", false},
{"workflow_dispatch stays on fallback", "workflow_dispatch", "", "", false},
}
for _, j := range advisoryBlacksmithRunnerJobs {
job := readCIWorkflow(t, j.file).job(t, j.job)
for _, c := range cases {
t.Run(j.file+"/"+j.job+"/"+c.name, func(t *testing.T) {
ctx := map[string]string{
"github.event_name": c.event,
"github.event.pull_request.head.repo.full_name": c.headRepo,
"github.repository": ownRepo,
"github.actor": c.actor,
}
want := j.fallback
if c.wantBlacksmith {
want = j.blacksmithLabel
}
if got := mustEvalGHRunsOn(t, job.RunsOn, ctx); got != want {
t.Errorf("%s/%s real runs-on %q evaluated under %+v = %q, want %q", j.file, j.job, job.RunsOn, c, got, want)
}
})
}
}
}
// --- Blacksmith cache-visibility precondition (spec-f7.md §2.2 Group B) ---
// blacksmithSetupGoCacheConsumers is every advisory job that restores the
// self-defined `blacksmith-sg-v1-` setup-go cache main.yml's
// blacksmith-setup-go-cache job seeds (B2, F7c implementation report).
var blacksmithSetupGoCacheConsumers = map[string][]string{
"regression.yml": {"regression"},
"migration-test.yml": {"historical-upgrades"},
"cross-version-smoke.yml": {"smoke"},
"proxied-local-smoke.yml": {"managed-local-smoke"},
}
// blacksmithSetupGoCacheKeyNamespace is the self-defined cache key prefix
// (not setup-go's own implicit key) that the seeder and every consumer share,
// so the "no save in a consumer job" checks below can scope to exactly this
// cache without also flagging an unrelated, legitimately-caching step (e.g.
// migration-test.yml's historical-dolt-* cache, which is a distinct
// restore/save pair of its own - see advisoryBinaryCaches below).
const blacksmithSetupGoCacheKeyNamespace = "blacksmith-sg-v1-"
// TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers pins that main.yml's
// blacksmith-setup-go-cache job exists, restores whatever is already cached,
// unconditionally re-runs every warm-up command (not gated on a cache hit,
// since a stale or partial restore must still self-heal), and then always
// saves - the "always warm, always save" design B2 requires so this job can
// safely be the ONLY writer of the Blacksmith-side setup-go cache every
// advisory consumer below reads from.
func TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers(t *testing.T) {
job := readCIWorkflow(t, "main.yml").job(t, "blacksmith-setup-go-cache")
if !strings.Contains(job.RunsOn, "blacksmith-") {
t.Errorf("main.yml's blacksmith-setup-go-cache runs-on = %q, want a Blacksmith label", job.RunsOn)
}
if job.TimeoutMinutes == 0 {
t.Error("main.yml's blacksmith-setup-go-cache has no timeout-minutes")
}
setupGoIndex := -1
for i, step := range job.Steps {
if actionFamily(step.Uses) != setupGoActionFamily {
continue
}
setupGoIndex = i
if step.ID != "setup-go" {
t.Errorf("main.yml's blacksmith-setup-go-cache setup-go step has id %q, want \"setup-go\"", step.ID)
}
// The seeder disables setup-go's OWN implicit cache (which would try
// to use a GitHub-hosted cache entry Blacksmith can't see) in favor of
// the explicit, self-keyed restore/save steps below.
if step.With["cache"] != "false" {
t.Errorf("main.yml's blacksmith-setup-go-cache setup-go cache = %q, want \"false\" (this job manages its own cache explicitly)", step.With["cache"])
}
if step.With["go-version-file"] != "go.mod" {
t.Errorf("main.yml's blacksmith-setup-go-cache setup-go go-version-file = %q, want go.mod", step.With["go-version-file"])
}
}
if setupGoIndex < 0 {
t.Fatal("main.yml's blacksmith-setup-go-cache has no actions/setup-go step")
}
restore := job.step(t, "Restore Blacksmith setup-go cache")
if actionFamily(restore.Uses) != cacheRestoreActionFamily {
t.Errorf("main.yml's blacksmith-setup-go-cache Restore step uses %q, want family %q", restore.Uses, cacheRestoreActionFamily)
}
if restore.If != "" {
t.Errorf("main.yml's blacksmith-setup-go-cache Restore step has if=%q, want unconditional (push-to-main only job, always trusted)", restore.If)
}
if !strings.HasPrefix(restore.With["key"], blacksmithSetupGoCacheKeyNamespace) {
t.Errorf("main.yml's blacksmith-setup-go-cache Restore key = %q, want it to start with %q", restore.With["key"], blacksmithSetupGoCacheKeyNamespace)
}
wantWarmups := []string{
"make build",
"go test -c -tags regression,gms_pure_go ./tests/regression",
}
for _, want := range wantWarmups {
found := false
for i, step := range job.Steps[setupGoIndex+1:] {
if strings.TrimSpace(step.Run) != want {
continue
}
found = true
// Unconditional, not gated on a cache hit (F7c review fix B2):
// the whole point of this job is to keep the cache warm, so it
// must always repopulate GOCACHE/GOMODCACHE.
if step.If != "" {
t.Errorf("main.yml's blacksmith-setup-go-cache step %q (index %d) has if=%q, want it unconditional",
want, setupGoIndex+1+i, step.If)
}
}
if !found {
t.Errorf("main.yml's blacksmith-setup-go-cache has no step that runs exactly %q", want)
}
}
save := job.step(t, "Save Blacksmith setup-go cache")
if actionFamily(save.Uses) != cacheSaveActionFamily {
t.Errorf("main.yml's blacksmith-setup-go-cache Save step uses %q, want family %q", save.Uses, cacheSaveActionFamily)
}
if save.If != "always()" {
t.Errorf("main.yml's blacksmith-setup-go-cache Save step has if=%q, want \"always()\" (save even if a warm-up step above failed)", save.If)
}
if !strings.HasPrefix(save.With["key"], blacksmithSetupGoCacheKeyNamespace) {
t.Errorf("main.yml's blacksmith-setup-go-cache Save key = %q, want it to start with %q", save.With["key"], blacksmithSetupGoCacheKeyNamespace)
}
if save.With["key"] != restore.With["key"] {
t.Errorf("main.yml's blacksmith-setup-go-cache Save key = %q, Restore key = %q; the seeder must save under the exact key it restores from", save.With["key"], restore.With["key"])
}
}
// TestBlacksmithSeederGuardedAgainstPullRequest pins main.yml's
// blacksmith-setup-go-cache job `if:` byte-for-byte (F7c review fix S2): this
// job is the ONLY writer every advisory consumer's blacksmith-sg-v1- restore
// trusts, so it must stay push-to-main-only even in the counterfactual where
// main.yml's `on:` trigger set grows a pull_request (or merge_group) entry
// someday. A guard that only checks github.repository (the pre-fix state)
// would not catch that: every same-repo PR also satisfies
// github.repository == 'gastownhall/beads'. The real regression this closes
// is evaluated below via evalGHExpr, not just a string match, so the job is
// also proven actually unreachable under a same-repo pull_request event.
func TestBlacksmithSeederGuardedAgainstPullRequest(t *testing.T) {
const wantIf = "github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads'"
job := readCIWorkflow(t, "main.yml").job(t, "blacksmith-setup-go-cache")
if job.If != wantIf {
t.Fatalf("main.yml's blacksmith-setup-go-cache if=%q, want exactly %q", job.If, wantIf)
}
const ownRepo = "gastownhall/beads"
cases := []struct {
name string
ctx map[string]string
want bool
}{
{"actual push to main", map[string]string{
"github.event_name": "push", "github.ref": "refs/heads/main", "github.repository": ownRepo,
}, true},
{"same-repo pull_request stays excluded", map[string]string{
"github.event_name": "pull_request", "github.ref": "refs/pull/1/merge", "github.repository": ownRepo,
"github.event.pull_request.head.repo.full_name": ownRepo,
}, false},
{"merge_group stays excluded", map[string]string{
"github.event_name": "merge_group", "github.ref": "refs/heads/gh-readonly-queue/main/pr-1", "github.repository": ownRepo,
}, false},
{"push to a non-main branch stays excluded", map[string]string{
"github.event_name": "push", "github.ref": "refs/heads/not-main", "github.repository": ownRepo,
}, false},
{"push from a fork stays excluded", map[string]string{
"github.event_name": "push", "github.ref": "refs/heads/main", "github.repository": "someone-else/beads",
}, false},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := evalGHExpr(job.If, c.ctx)
if err != nil {
t.Fatalf("evalGHExpr(%q): %v", job.If, err)
}
if ghTruthy(got) != c.want {
t.Errorf("evalGHExpr(%q) under %+v = %#v, want truthy=%v", job.If, c.ctx, got, c.want)
}
})
}
}
// TestMainWorkflowHasNoSameRepoPRReachableTrigger is the direct half of F7c
// review fix S2's "pin the seeder's trust boundary" ask: belt-and-suspenders
// alongside the job-level guard in TestBlacksmithSeederGuardedAgainstPullRequest.
// The guard defuses the vulnerability even if one of these triggers is added
// to main.yml's `on:` block, but this test catches the trigger addition
// itself at the workflow-trust-surface level, so a future edit here gets
// flagged before anyone has to reason about whether the job-level `if:`
// still holds. (A mutation that only edits `on:`, leaving the job-level `if:`
// untouched, would otherwise not fail any other test here.)
func TestMainWorkflowHasNoSameRepoPRReachableTrigger(t *testing.T) {
type mainTriggers struct {
On map[string]any `yaml:"on"`
}
var parsed mainTriggers
text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/main.yml")
if err := yaml.Unmarshal([]byte(text), &parsed); err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"pull_request", "pull_request_target", "merge_group", "workflow_run"} {
if _, ok := parsed.On[forbidden]; ok {
t.Errorf("main.yml's on: has a %q trigger; its blacksmith-setup-go-cache seeder is the ONLY writer every advisory consumer trusts, so this workflow must never become reachable from a same-repo PR/merge-queue event even with the job-level guard as a second layer", forbidden)
}
}
if _, ok := parsed.On["push"]; !ok {
t.Errorf("main.yml's on: has no push trigger: %+v", parsed.On)
}
}
// TestAdvisoryBlacksmithConsumersAreCacheRestoreOnly replaces the pre-review
// TestAdvisoryBlacksmithConsumersKeepImplicitSetupGoCache (F7c review fix
// B2): each consumer must disable setup-go's own implicit cache on a
// self-hosted (Blacksmith) runner and restore-only from the self-keyed
// `blacksmith-sg-v1-` namespace, with NO save step in that namespace - only
// main.yml's seeder may ever write it, so a same-repo PR run can read the
// cache but never poison what another PR or main's seeder reads back. The
// "no save" check is scoped to the blacksmith-sg-v1- key namespace
// specifically (not "no actions/cache/save in this job at all"), since
// migration-test.yml's historical-upgrades job keeps its own historical-dolt-*
// cache in a different namespace - governed by its own restore/save-gated
// pair below (advisoryBinaryCaches / F7c review fix X1), not an exemption
// from this one.
func TestAdvisoryBlacksmithConsumersAreCacheRestoreOnly(t *testing.T) {
for file, jobNames := range blacksmithSetupGoCacheConsumers {
workflow := readCIWorkflow(t, file)
for _, jobName := range jobNames {
t.Run(file+"/"+jobName, func(t *testing.T) {
job := workflow.job(t, jobName)
var sawSetupGo, sawRestore bool
for _, step := range job.Steps {
switch actionFamily(step.Uses) {
case setupGoActionFamily:
sawSetupGo = true
// F7c review fix N1: fail-closed. Pinned to the
// "== 'github-hosted'" form (not "!= 'self-hosted'")
// so an empty/unknown runner.environment value keeps
// caching OFF instead of turning it on.
if step.With["cache"] != "${{ runner.environment == 'github-hosted' }}" {
t.Errorf("%s job %s setup-go cache = %q, want it disabled on self-hosted runners (fail-closed)", file, jobName, step.With["cache"])
}
case cacheRestoreActionFamily:
if strings.HasPrefix(step.With["key"], blacksmithSetupGoCacheKeyNamespace) {
sawRestore = true
if !strings.Contains(step.If, "runner.environment == 'self-hosted'") {
t.Errorf("%s job %s setup-go cache Restore step has if=%q, want it gated on runner.environment == 'self-hosted'", file, jobName, step.If)
}
}
case cacheSaveActionFamily, cacheMonolithicActionFamily:
if strings.HasPrefix(step.With["key"], blacksmithSetupGoCacheKeyNamespace) {
t.Errorf("%s job %s has a %s step keyed in the blacksmith-sg-v1- namespace (%q); only main.yml's seeder may save this cache", file, jobName, actionFamily(step.Uses), step.With["key"])
}
}
}
if !sawSetupGo {
t.Errorf("%s job %s has no actions/setup-go step", file, jobName)
}
if !sawRestore {
t.Errorf("%s job %s has no blacksmith-sg-v1- cache restore step", file, jobName)
}
})
}
}
}
// TestBlacksmithSeederCacheKeyIsNotPerCommit pins F7c review fix S5: the
// seeder's save/restore key suffix must be a UTC calendar day
// (steps.cache-date.outputs.today), not github.sha. Keying per commit meant
// every single push to main - whether or not go.sum changed - wrote a new
// multi-GB ~/go/pkg/mod + ~/.cache/go-build entry under this namespace; on a
// cache store with LRU eviction (Blacksmith) that churn risked evicting
// F7a's own beads-go-mod-v2-*/beads-go-build-v2-* entries. Keying per day
// instead caps writes to at most once per calendar day while still picking
// up a go.sum change on the very next push.
func TestBlacksmithSeederCacheKeyIsNotPerCommit(t *testing.T) {
job := readCIWorkflow(t, "main.yml").job(t, "blacksmith-setup-go-cache")
dateStep := job.step(t, "Compute cache date")
if dateStep.ID != "cache-date" {
t.Errorf("main.yml's blacksmith-setup-go-cache Compute cache date step has id %q, want \"cache-date\"", dateStep.ID)
}
if !strings.Contains(dateStep.Run, "date -u") {
t.Errorf("main.yml's blacksmith-setup-go-cache Compute cache date step run = %q, want it to compute a UTC date", dateStep.Run)
}
restore := job.step(t, "Restore Blacksmith setup-go cache")
save := job.step(t, "Save Blacksmith setup-go cache")
for name, step := range map[string]ciWorkflowStep{"Restore": restore, "Save": save} {
key := step.With["key"]
if strings.Contains(key, "github.sha") {
t.Errorf("main.yml's blacksmith-setup-go-cache %s step key = %q, must not key per-commit (github.sha) - see F7c review fix S5", name, key)
}
if !strings.Contains(key, "steps.cache-date.outputs.today") {
t.Errorf("main.yml's blacksmith-setup-go-cache %s step key = %q, want it keyed by steps.cache-date.outputs.today", name, key)
}
}
}
// TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers pins that every
// consumer's restore key/restore-keys are textually identical to main.yml
// seeder's save key (F7c review fix B2, closes a seeder/consumer key-mismatch
// mutation): if a consumer's key format ever drifted from the seeder's (a
// different hash segment order, a missing runner.arch, etc.) the seeder would
// keep writing entries no consumer could ever restore, silently degrading
// every advisory Blacksmith job back to a cold cache.
func TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers(t *testing.T) {
seeder := readCIWorkflow(t, "main.yml").job(t, "blacksmith-setup-go-cache")
seederSave := seeder.step(t, "Save Blacksmith setup-go cache")
seederKey := seederSave.With["key"]
if seederKey == "" {
t.Fatal("main.yml's blacksmith-setup-go-cache Save step has no key")
}
for file, jobNames := range blacksmithSetupGoCacheConsumers {
workflow := readCIWorkflow(t, file)
for _, jobName := range jobNames {
job := workflow.job(t, jobName)
var restore *ciWorkflowStep
for i := range job.Steps {
step := &job.Steps[i]
if actionFamily(step.Uses) == cacheRestoreActionFamily && strings.HasPrefix(step.With["key"], blacksmithSetupGoCacheKeyNamespace) {
restore = step
break
}
}
if restore == nil {
t.Errorf("%s job %s has no blacksmith-sg-v1- restore step", file, jobName)
continue
}
if restore.With["key"] != seederKey {
t.Errorf("%s job %s restore key = %q, want it identical to the seeder's save key %q", file, jobName, restore.With["key"], seederKey)
}
if !strings.Contains(restore.With["restore-keys"], strings.TrimSuffix(seederKey, "${{ steps.cache-date.outputs.today }}")) {
t.Errorf("%s job %s restore-keys %q does not contain the seeder's key prefix (without the commit-specific suffix)", file, jobName, restore.With["restore-keys"])
}
}
}
}
// --- Binary caches: restore-always, save only off pull_request ------------
// advisoryBinaryCaches are the per-binary caches F7c review fixes B2 and X1
// converted from a monolithic (auto-saving) actions/cache into an explicit
// restore/save pair, so a same-repo PR can read a previously published
// binary but never publish its own into a cache another run would trust
// unverified. migration-test.yml's historical-dolt-* cache was originally
// exempted here on the theory that scripts/migration-test/lib/binary.sh's
// sha256 verification of every extracted archive made a bare, auto-saving
// actions/cache safe regardless of who wrote it. That exemption was unsound
// (F7c review fix X1): `tar -P` extraction plus lib/binary.sh's `cp -f`
// following a symlink planted inside the archive can redirect the final copy
// to an arbitrary path (e.g. over the checked-out workspace or the candidate
// binary itself) before the checksum check ever runs, so checksum
// verification alone does not make a same-repo-PR-writable cache entry safe
// to trust. historical-dolt-* now gets the same restore-always/save-off-PR
// split as the other two.
var advisoryBinaryCaches = []struct {
file, job, restoreStep, saveStep, keyPrefix string
// wantSaveIf is the save step's `if:` condition, required byte-for-byte
// (F7c review fix S3): a `strings.Contains` check here would pass under
// e.g. `... || true`, which always evaluates true and silently
// reintroduces the same-repo-PR poisoning path this whole table exists
// to close.
wantSaveIf string
}{
{"cross-version-smoke.yml", "smoke", "Restore previous release binaries cache", "Save previous release binaries cache", "smoke-binaries-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"},
{"regression.yml", "regression", "Restore baseline binary cache", "Save baseline binary cache", "regression-baseline-", "steps.detect.outputs.run_regression == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')"},
{"migration-test.yml", "historical-upgrades", "Restore pinned historical release cache", "Save pinned historical release cache", "historical-dolt-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"},
}
// TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated pins the restore/save
// split itself (F7c review fix B2): the restore step always runs (modulo the
// job's own pre-existing gate, e.g. regression's run_regression detection),
// the save step additionally requires an exact allow-list of
// `github.event_name == 'push' || github.event_name == 'workflow_dispatch'`
// (F7c review fix, discovered via the S2 sweep: a deny-list of
// `!= 'pull_request'` also admits a hypothetical future merge_group event,
// which each of these three jobs' runs-on expressions already resolve to
// Blacksmith for), both steps key off the same cache, and no monolithic
// (bare) actions/cache step remains for either binary cache - a monolithic
// step would silently reintroduce the auto-save-on-any-PR poisoning path B2
// closes.
func TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated(t *testing.T) {
for _, c := range advisoryBinaryCaches {
t.Run(c.file, func(t *testing.T) {
workflow := readCIWorkflow(t, c.file)
job := workflow.job(t, c.job)
restore := job.step(t, c.restoreStep)
if actionFamily(restore.Uses) != cacheRestoreActionFamily {
t.Errorf("%s job %s step %q uses %q, want family %q", c.file, c.job, c.restoreStep, restore.Uses, cacheRestoreActionFamily)
}
if !strings.HasPrefix(restore.With["key"], c.keyPrefix) {
t.Errorf("%s job %s step %q key = %q, want prefix %q", c.file, c.job, c.restoreStep, restore.With["key"], c.keyPrefix)
}
save := job.step(t, c.saveStep)
if actionFamily(save.Uses) != cacheSaveActionFamily {
t.Errorf("%s job %s step %q uses %q, want family %q", c.file, c.job, c.saveStep, save.Uses, cacheSaveActionFamily)
}
if save.If != c.wantSaveIf {
t.Errorf("%s job %s step %q has if=%q, want exactly %q", c.file, c.job, c.saveStep, save.If, c.wantSaveIf)
}
if save.With["key"] != restore.With["key"] {
t.Errorf("%s job %s: restore key %q != save key %q", c.file, c.job, restore.With["key"], save.With["key"])
}
for _, step := range job.Steps {
if actionFamily(step.Uses) == cacheMonolithicActionFamily && strings.HasPrefix(step.With["key"], c.keyPrefix) {
t.Errorf("%s job %s has a monolithic actions/cache step keyed %q; B2 requires an explicit restore/save split here", c.file, c.job, step.With["key"])
}
}
})
}
}
// TestRegressionStepsAfterDetectAreGated pins that every step after
// regression.yml's "Decide whether to run differential regression tests"
// (id: detect) stays gated on steps.detect.outputs.run_regression == 'true'
// (F7c review fix S1, closes an "ungated regression step" mutation): the
// fold that moved detect-regression-need into this job's own first step
// means a single dropped `if:` would make an inapplicable PR silently pay
// for (or worse, run) the rest of the job instead of skipping it.
func TestRegressionStepsAfterDetectAreGated(t *testing.T) {
job := readCIWorkflow(t, "regression.yml").job(t, "regression")
detectIndex := job.stepIndex(t, "Decide whether to run differential regression tests")
const want = "steps.detect.outputs.run_regression == 'true'"
for _, step := range job.Steps[detectIndex+1:] {
if !strings.Contains(step.If, want) {
t.Errorf("regression.yml step %q has if=%q, want it to contain %q", step.Name, step.If, want)
}
}
}
// TestRegressionDetectStepBehavioral runs regression.yml's "Decide whether to
// run differential regression tests" (id: detect) step for real, the same
// way TestMigrationHarnessLoopExecutesBehaviorally exercises the migration
// loop (F7c review fix S1, closes mutation M13: a detect step hardcoded to
// always emit run_regression=false). TestRegressionStepsAfterDetectAreGated
// only pins that steps AFTER detect stay gated on its output; it says nothing
// about whether detect's own logic ever produces "true", so a detect step
// that always reports false would make every downstream gate vacuously
// satisfied while regression silently never ran. This executes the step's
// actual bash against real GITHUB_OUTPUT/GITHUB_EVENT_PATH files and a real
// two-commit git repo for each input that must change the verdict.
func TestRegressionDetectStepBehavioral(t *testing.T) {
requireHostTool(t, "bash")
requireHostTool(t, "jq")
requireHostTool(t, "git")
job := readCIWorkflow(t, "regression.yml").job(t, "regression")
step := job.step(t, "Decide whether to run differential regression tests")
if step.ID != "detect" {
t.Fatalf("detect step id = %q, want %q", step.ID, "detect")
}
// newRepo creates a base commit, then (if any changedFiles are given) a
// second commit that adds each of them, and returns the repo dir plus
// both commit SHAs for PR_BASE_SHA/PR_HEAD_SHA.
newRepo := func(t *testing.T, changedFiles ...string) (dir, base, head string) {
t.Helper()
dir = t.TempDir()
git := func(args ...string) string {
t.Helper()
cmd := exec.Command("git", args...)
cmd.Dir = dir
cmd.Env = append(os.Environ(),
"GIT_CONFIG_NOSYSTEM=1",
"GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t.test",
"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t.test",
)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
return strings.TrimSpace(string(out))
}
git("init", "-q", "-b", "main")
if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte("base\n"), 0o644); err != nil {
t.Fatal(err)
}
git("add", "README.md")
git("commit", "-q", "-m", "base")
base = git("rev-parse", "HEAD")
for _, f := range changedFiles {
full := filepath.Join(dir, f)
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte("x\n"), 0o644); err != nil {
t.Fatal(err)
}
git("add", f)
}
if len(changedFiles) > 0 {
git("commit", "-q", "-m", "head")
}
head = git("rev-parse", "HEAD")
return dir, base, head
}
writeEvent := func(t *testing.T, dir string, labels ...string) string {
t.Helper()
type label struct {
Name string `json:"name"`
}
type prBody struct {
Labels []label `json:"labels"`
}
type event struct {
PullRequest prBody `json:"pull_request"`
}
e := event{PullRequest: prBody{Labels: []label{}}}
for _, l := range labels {
e.PullRequest.Labels = append(e.PullRequest.Labels, label{Name: l})
}
b, err := json.Marshal(e)
if err != nil {
t.Fatal(err)
}
p := filepath.Join(dir, "event.json")
if err := os.WriteFile(p, b, 0o644); err != nil {
t.Fatal(err)
}
return p
}
runDetect := func(t *testing.T, dir, eventPath, eventName, base, head string) (runRegression, reason string) {
t.Helper()
outFile := filepath.Join(t.TempDir(), "output")
if err := os.WriteFile(outFile, nil, 0o644); err != nil {
t.Fatal(err)
}
cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step.Run)
cmd.Dir = dir
cmd.Env = append(os.Environ(),
"EVENT_NAME="+eventName,
"PR_BASE_SHA="+base,
"PR_HEAD_SHA="+head,
"GITHUB_OUTPUT="+outFile,
"GITHUB_EVENT_PATH="+eventPath,
)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("detect step: %v\noutput:\n%s", err, out)
}
outBytes, err := os.ReadFile(outFile)
if err != nil {
t.Fatal(err)
}
for _, line := range strings.Split(string(outBytes), "\n") {
if v, ok := strings.CutPrefix(line, "run_regression="); ok {
runRegression = v
}
if v, ok := strings.CutPrefix(line, "reason="); ok {
reason = v
}
}
return runRegression, reason
}
cases := []struct {
name string
eventName string
labels []string
changedFiles []string
want string
}{
{"push always runs", "push", nil, nil, "true"},
{"run-regression label forces true", "pull_request", []string{"run-regression"}, []string{"docs/unrelated.md"}, "true"},
{"skip-regression label forces false", "pull_request", []string{"skip-regression"}, []string{"cmd/bd/x.go"}, "false"},
{"cmd/bd non-test change runs", "pull_request", nil, []string{"cmd/bd/x.go"}, "true"},
{"only a cmd/bd test file changed skips", "pull_request", nil, []string{"cmd/bd/x_test.go"}, "false"},
{"docs-only change skips", "pull_request", nil, []string{"docs/a.md"}, "false"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
dir, base, head := newRepo(t, c.changedFiles...)
eventPath := writeEvent(t, dir, c.labels...)
got, reason := runDetect(t, dir, eventPath, c.eventName, base, head)
if got != c.want {
t.Errorf("run_regression = %q (reason %q), want %q", got, reason, c.want)
}
})
}
}
// --- General sweep: no Blacksmith-reachable advisory job may save a cache --
// generalCacheSweepWorkflows is every F7c advisory workflow plus main.yml
// (the seeder). pr.yml, pr-risk.yml and bazel.yml also run jobs on
// Blacksmith, but their Go-cache topology is already pinned exhaustively,
// job-by-job and step-by-step, by TestGoCacheOwnershipTopology and
// TestBazelWorkflowCacheTopology; those jobs' runs-on expressions also
// depend on a prior job's `needs.rbe.outputs.enabled` output, which the
// minimal evalGHExpr engine below (deliberately scoped to github.*/runner.*
// context lookups) cannot resolve, so folding them into this sweep would
// either silently under-check them or require duplicating that existing
// machinery. This list is every workflow this F7c round can actually edit
// plus the one workflow (main.yml) whose seeder job this round added a new
// guard to.
var generalCacheSweepWorkflows = append(mapKeys(blacksmithSetupGoCacheConsumers), "main.yml")
// blacksmithTrustContexts are the two event shapes a same-repo Blacksmith
// `runs-on` ternary can route onto a `blacksmith-*` label for (F7c review fix
// S2): a trusted same-repo pull_request, and merge_group. runner.environment
// is pinned to "self-hosted" in both, matching what a job actually observes
// once it lands on a Blacksmith runner, so a step's own
// `runner.environment == 'self-hosted'`-gated logic evaluates the same way
// here as it would for real.
var blacksmithTrustContexts = map[string]map[string]string{
"pull_request": {
"github.event_name": "pull_request",
"github.event.pull_request.head.repo.full_name": "gastownhall/beads",
"github.repository": "gastownhall/beads",
"github.actor": "alice",
"github.ref": "refs/pull/1/merge",
"runner.environment": "self-hosted",
},
"merge_group": {
"github.event_name": "merge_group",
"github.repository": "gastownhall/beads",
"github.ref": "refs/heads/gh-readonly-queue/main/pr-1",
"runner.environment": "self-hosted",
},
}
// ghExprTruthyDefaultTrue evaluates expr under ctx, treating both an empty
// expr and an evaluation error as truthy/reachable: this sweep's job is to
// catch a missing or wrong save-gate, so an `if:` this minimal evaluator
// cannot parse must fail closed (assume the step/job runs) rather than
// silently skip the jobs or steps that use it.
func ghExprTruthyDefaultTrue(expr string, ctx map[string]string) bool {
if strings.TrimSpace(expr) == "" {
return true
}
v, err := evalGHExpr(expr, ctx)
if err != nil {
return true
}
return ghTruthy(v)
}
// resolveRunsOnLabel returns the runner label a job's runs-on resolves to
// under ctx: the literal string itself if it is not a `${{ ... }}`
// expression, the evaluated result if it is and evaluates to a string, or ""
// (never a Blacksmith label) if it cannot be resolved at all.
func resolveRunsOnLabel(runsOn string, ctx map[string]string) string {
if !strings.Contains(runsOn, "${{") {
return strings.TrimSpace(runsOn)
}
v, err := evalGHExpr(runsOn, ctx)
if err != nil {
return ""
}
s, ok := v.(string)
if !ok {
return ""
}
return s
}
// TestBlacksmithReachableAdvisoryJobsNeverSaveACache is the general sweep the
// F7c re-review required (S2): rather than a hand-picked table of the caches
// known about so far (B2, X1), evaluate EVERY job in EVERY generalCacheSweep-
// Workflows workflow against both blacksmithTrustContexts, and for every job
// whose runs-on actually resolves to a blacksmith-* label under one of them,
// forbid: a bare (monolithic) actions/cache step; an actions/cache/save step
// that is not gated off (would still run under) that same context; a
// setup-go step whose effective `cache` input is not disabled on a
// self-hosted runner; and a setup-node/setup-python step with any `cache`
// input set at all. This is the mechanism that would have caught N3 (docs
// setup-node cache: npm), N4 (a new bare actions/cache step in a Blacksmith
// job), and - once main.yml's seeder gained its S2 job-level guard - proves
// X1/N6 (main.yml growing a pull_request trigger) still cannot make the
// seeder's always-on save step run on a PR, without any of those needing
// their own bespoke test.
func TestBlacksmithReachableAdvisoryJobsNeverSaveACache(t *testing.T) {
// Sorted so t.Run subtest names (and therefore failure ordering) are
// stable regardless of Go's randomized map iteration.
ctxNames := make([]string, 0, len(blacksmithTrustContexts))
for name := range blacksmithTrustContexts {
ctxNames = append(ctxNames, name)
}
sort.Strings(ctxNames)
for _, file := range generalCacheSweepWorkflows {
workflow := readCIWorkflow(t, file)
for jobName, job := range workflow.Jobs {
job := job
for _, ctxName := range ctxNames {
ctx := blacksmithTrustContexts[ctxName]
t.Run(file+"/"+jobName+"/"+ctxName, func(t *testing.T) {
if !ghExprTruthyDefaultTrue(job.If, ctx) {
return // job cannot even run under this event
}
label := resolveRunsOnLabel(job.RunsOn, ctx)
if !strings.HasPrefix(label, "blacksmith-") {
return // doesn't land on Blacksmith under this context
}
// Every context under which the job is Blacksmith-reachable
// must be checked independently (not just the first one
// found): a save gate like `github.event_name !=
// 'pull_request'` is false under pull_request but true
// under merge_group, so a job reachable on Blacksmith
// under both needs both verified.
for _, step := range job.Steps {
family := actionFamily(step.Uses)
switch family {
case cacheMonolithicActionFamily:
t.Errorf("%s job %s step %q uses bare actions/cache on a Blacksmith-reachable job; it auto-saves on any key miss (B2/X1 forbid this - use actions/cache/restore + a non-PR-gated actions/cache/save)", file, jobName, step.Name)
case cacheSaveActionFamily:
if ghExprTruthyDefaultTrue(step.If, ctx) {
t.Errorf("%s job %s step %q (actions/cache/save) has if=%q, which still runs under a same-repo Blacksmith %s event; it must be gated off", file, jobName, step.Name, step.If, ctxName)
}
case setupGoActionFamily:
cacheVal, ok := step.With["cache"]
wouldCache := true
if ok {
if !strings.Contains(cacheVal, "${{") {
wouldCache = cacheVal == "true"
} else if v, err := evalGHExpr(cacheVal, ctx); err == nil {
wouldCache = ghTruthy(v)
}
}
if wouldCache {
t.Errorf("%s job %s setup-go step %q has cache=%q, which stays enabled on a self-hosted (Blacksmith) runner; it must disable its own implicit cache there", file, jobName, step.Name, cacheVal)
}
case setupNodeActionFamily, setupPythonActionFamily:
if cacheVal, ok := step.With["cache"]; ok && cacheVal != "" {
t.Errorf("%s job %s step %q sets cache=%q on a Blacksmith-reachable job; setup-node/setup-python's own cache uses a bare actions/cache internally", file, jobName, step.Name, cacheVal)
}
}
}
})
}
}
}
}