beads/scripts/ci_f7c_advisory_test.go
Julian Knutsen 77ab980790
ci(bazel): run the hook-timeout-backends check under Bazel; test-nix stops running tests (#7357)
checks.nix's hook-timeout-backends (the tracked git hook shims against
real GNU/uutils/busybox/toybox timeout implementations, with and without
perl, under dash, bash and busybox ash: 240 cases) was the one test PR
Risk's test-nix ran on a GitHub runner, through `nix flake check`. It is
now //tests/hook_timeout_backends:hook_timeout_backends_test, an sh_test
(pr-core-only) that the bazel-test lane runs remotely on every PR: the
static uutils 0.12.0, busybox 1.35.0 and toybox 0.8.15 binaries come in
sha256-pinned through MODULE.bazel; dash, bash, perl and GNU coreutils
from the executor, failing (not skipping) when one is missing.

checks.nix goes and flake.nix declares no checks, so test-nix's
`nix run .#default` + `nix flake check` is a packaging check (build the
package Nix users install, evaluate every flake output), not a test.
TestHookTimeoutBackendsRunUnderBazel pins the target and that no flake
check comes back.

ga-96smfk.22

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-07 11:05:39 +00:00

485 lines
20 KiB
Go

package scripts_test
import (
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
// F7c (spec-f7.md §2.4, §4.3): advisory workflows moved onto a same-repo-PR
// Blacksmith runner. These tests pin the invariants that make that safe: no
// advisory job can read a secret just because it now names a Blacksmith
// label, and no Blacksmith-reachable job saves a cache. The path-filtered upgrade suites F7c folded
// (Migration Test Harness, Cross-Version Smoke) and their filter and fold
// checks are gone: those suites run under Bazel (//tests/migration,
// //tests/upgrade_smoke).
type pullRequestPaths struct {
On struct {
PullRequest struct {
Paths []string `yaml:"paths"`
} `yaml:"pull_request"`
} `yaml:"on"`
}
func readPullRequestPaths(t *testing.T, file string) []string {
t.Helper()
var parsed pullRequestPaths
text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+file)
if err := yaml.Unmarshal([]byte(text), &parsed); err != nil {
t.Fatalf("parse %s: %v", file, err)
}
return parsed.On.PullRequest.Paths
}
type pushPaths struct {
On struct {
Push struct {
Paths []string `yaml:"paths"`
} `yaml:"push"`
} `yaml:"on"`
}
func readPushPaths(t *testing.T, file string) []string {
t.Helper()
var parsed pushPaths
text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+file)
if err := yaml.Unmarshal([]byte(text), &parsed); err != nil {
t.Fatalf("parse %s: %v", file, err)
}
return parsed.On.Push.Paths
}
// TestNixBuildDropsPullRequestTriggerNotPushOrDispatch pins the one
// "delete the pull_request trigger" trigger change in F7c: nix-build.yml's
// PR coverage is fully redundant with PR Risk's required test-nix job (which
// runs `nix run .#default` plus `nix flake check -L`, which evaluates every
// flake output, on every PR: a superset of `nix build .#default`), but push
// and workflow_dispatch must survive so
// the plain `nix build` path stays covered post-merge.
func TestNixBuildDropsPullRequestTriggerNotPushOrDispatch(t *testing.T) {
type nixTriggers struct {
On map[string]any `yaml:"on"`
}
var parsed nixTriggers
text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/nix-build.yml")
if err := yaml.Unmarshal([]byte(text), &parsed); err != nil {
t.Fatal(err)
}
if _, ok := parsed.On["pull_request"]; ok {
t.Errorf("nix-build.yml still has a pull_request trigger; PR Risk's test-nix job is a superset (spec-f7.md §2.4)")
}
push, ok := parsed.On["push"].(map[string]any)
if !ok {
t.Fatalf("nix-build.yml has no push trigger: %+v", parsed.On)
}
branches, _ := push["branches"].([]any)
var branchNames []string
for _, b := range branches {
branchNames = append(branchNames, fmt.Sprint(b))
}
if !contains(branchNames, "main") {
t.Errorf("nix-build.yml must still push on main: %+v", push)
}
if _, ok := parsed.On["workflow_dispatch"]; !ok {
t.Errorf("nix-build.yml must still support workflow_dispatch")
}
// PR Risk's test-nix must actually be the superset this removal leans on:
// it has to build the default package AND run the flake checks, not just
// one of the two.
prRisk := readCIWorkflow(t, "pr-risk.yml")
testNix := prRisk.job(t, "test-nix")
var sawBuild, sawFlakeCheck bool
for _, step := range testNix.Steps {
if strings.Contains(step.Run, "nix run .#default") {
sawBuild = true
}
if strings.Contains(step.Run, "nix flake check") {
sawFlakeCheck = true
}
}
if !sawBuild {
t.Error("pr-risk.yml's test-nix no longer builds/runs the default package; nix-build.yml's pull_request trigger would need to come back")
}
if !sawFlakeCheck {
t.Error("pr-risk.yml's test-nix no longer runs the flake checks")
}
}
// The flake's one test check (hook-timeout-backends, which only PR Risk's
// test-nix ran) is a Bazel target the PR-core lane runs on every PR, against
// all five tracked hooks and the three pinned static timeout
// implementations; the flake declares no checks, so test-nix's
// `nix flake check` is a packaging check (an evaluation of the outputs), not
// a test run on a GitHub runner.
func TestHookTimeoutBackendsRunUnderBazel(t *testing.T) {
root := sourceRepoRoot(t)
if _, err := os.Stat(filepath.Join(root, "checks.nix")); err == nil {
t.Error("checks.nix is back; hook-timeout-backends runs under Bazel (//tests/hook_timeout_backends)")
}
if flake := readPolicyFile(t, root, "flake.nix"); regexp.MustCompile(`(?m)^\s*checks\s*=`).MatchString(flake) {
t.Error("flake.nix declares checks; test checks run under Bazel, test-nix's nix flake check only evaluates the outputs")
}
rule := bazelRuleBlock(readPolicyFile(t, root, "tests/hook_timeout_backends/BUILD.bazel"), "hook_timeout_backends_test")
for _, want := range []string{
`srcs = ["hook_timeout_backends_test.sh"],`,
`"$(rootpath @busybox_static//file)",`,
`"$(rootpath @toybox_static//file)",`,
`"$(rootpath @uutils_coreutils//:coreutils)",`,
`"$(rootpath //:.githooks/pre-commit)",`,
`"$(rootpath //:.githooks/post-merge)",`,
`"$(rootpath //:.githooks/pre-push)",`,
`"$(rootpath //:.githooks/post-checkout)",`,
`"$(rootpath //:.githooks/prepare-commit-msg)",`,
`tags = ["pr-core-only"],`,
} {
if !strings.Contains(rule, want) {
t.Errorf("//tests/hook_timeout_backends:hook_timeout_backends_test lacks %q:\n%s", want, rule)
}
}
hooks, err := filepath.Glob(filepath.Join(root, ".githooks", "*"))
if err != nil {
t.Fatal(err)
}
for _, hook := range hooks {
if !strings.Contains(rule, `"$(rootpath //:.githooks/`+filepath.Base(hook)+`)",`) {
t.Errorf("tracked hook .githooks/%s is not in hook_timeout_backends_test's args", filepath.Base(hook))
}
}
}
func sortedCopy(items []string) []string {
out := append([]string(nil), items...)
sort.Strings(out)
return out
}
// --- Security: no job gains a secret just by naming a Blacksmith label ----
// blacksmithAdvisoryWorkflows are every workflow file F7c moved a job onto a
// same-repo-PR (or push-only, for main.yml's seed) Blacksmith runner.
var blacksmithAdvisoryWorkflows = []string{
"docs-mintlify.yml",
"main.yml",
}
// TestBlacksmithAdvisoryJobsReadNoSecrets is the security invariant spec-f7.md
// §3 requires: "a new policy test asserts that no job whose runs-on names
// blacksmith- reads secrets.". pr-risk.yml's own no-secrets walk is separate
// and untouched; this one covers the F7c advisory workflows plus main.yml's
// new seed job.
func TestBlacksmithAdvisoryJobsReadNoSecrets(t *testing.T) {
for _, file := range blacksmithAdvisoryWorkflows {
workflow := readCIWorkflow(t, file)
for name, job := range workflow.Jobs {
if !strings.Contains(job.RunsOn, "blacksmith-") {
continue
}
t.Run(file+"/"+name, func(t *testing.T) {
if containsSecretRef(fmt.Sprint(job.Env)) {
t.Errorf("%s job %s's env references secrets.", file, name)
}
for _, step := range job.Steps {
if containsSecretRef(step.Run) {
t.Errorf("%s job %s step %q's run references secrets.", file, name, step.Name)
}
if containsSecretRef(fmt.Sprint(step.Env)) {
t.Errorf("%s job %s step %q's env references secrets.", file, name, step.Name)
}
if containsSecretRef(fmt.Sprint(step.With)) {
t.Errorf("%s job %s step %q's with references secrets.", file, name, step.Name)
}
}
})
}
}
}
func containsSecretRef(s string) bool {
return strings.Contains(s, "secrets.")
}
// advisoryBlacksmithRunnerJobs pins which F7c advisory job runs on which
// same-repo Blacksmith expression, and what label each falls back to when
// the same-repo/merge_group condition is false.
var advisoryBlacksmithRunnerJobs = []struct {
file string
job string
blacksmithLabel string
fallback string
}{
{"docs-mintlify.yml", "broken-links", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"},
}
// TestF7cAdvisorySameRepoBlacksmithExpressionSemantics evaluates each F7c
// advisory job's REAL, as-parsed-from-YAML runs-on expression text through
// F7a's shared evalGHExpr/mustEvalGHRunsOn (ci_blacksmith_runner_test.go),
// against a truth table covering every event shape the advisory workflows
// see: merge_group, a same-repo PR, a fork PR, a Dependabot PR, a deleted
// fork head, and push/pull_request_target/workflow_dispatch/schedule.
//
// This replaces an earlier, F7c-local hand-written Go mirror of the same
// boolean logic (a tautology risk the F7a review flagged for its own
// equivalent test: a change to both the mirror and the real expression, in
// the same wrong way, would still pass). Per the coordinator, this test now
// reuses F7a's real evaluator instead of building a second one, and is
// scoped to the F7c advisory jobs specifically; F7a's own
// TestSameRepoBlacksmithExpressionSemantics (ci_blacksmith_runner_test.go)
// covers the shared sameRepoBlacksmith{2,4,8}vcpu consts themselves.
func TestF7cAdvisorySameRepoBlacksmithExpressionSemantics(t *testing.T) {
const ownRepo = "steveyegge/beads"
type tc struct {
name string
event string
headRepo string // github.event.pull_request.head.repo.full_name; "" = fork or deleted fork head
actor string
wantBlacksmith bool
}
cases := []tc{
{"same-repo PR, human actor", "pull_request", ownRepo, "alice", true},
{"merge_group always Blacksmith", "merge_group", "", "", true},
{"fork PR stays on fallback", "pull_request", "someone-else/beads", "alice", false},
{"deleted fork head stays on fallback", "pull_request", "", "alice", false},
{"same-repo PR, dependabot actor stays on fallback", "pull_request", ownRepo, "dependabot[bot]", false},
{"push stays on fallback", "push", "", "alice", false},
{"pull_request_target stays on fallback", "pull_request_target", ownRepo, "alice", false},
{"schedule stays on fallback", "schedule", "", "", false},
{"workflow_dispatch stays on fallback", "workflow_dispatch", "", "", false},
}
for _, j := range advisoryBlacksmithRunnerJobs {
job := readCIWorkflow(t, j.file).job(t, j.job)
for _, c := range cases {
t.Run(j.file+"/"+j.job+"/"+c.name, func(t *testing.T) {
ctx := map[string]string{
"github.event_name": c.event,
"github.event.pull_request.head.repo.full_name": c.headRepo,
"github.repository": ownRepo,
"github.actor": c.actor,
}
want := j.fallback
if c.wantBlacksmith {
want = j.blacksmithLabel
}
if got := mustEvalGHRunsOn(t, job.RunsOn, ctx); got != want {
t.Errorf("%s/%s real runs-on %q evaluated under %+v = %q, want %q", j.file, j.job, job.RunsOn, c, got, want)
}
})
}
}
}
// --- Retired setup-go seeder (ga-96smfk.58) --------------------------------
// TestNoBlacksmithSetupGoCacheNamespace: main.yml's blacksmith-setup-go-cache
// seeder warmed the self-keyed blacksmith-sg-v1- cache for the F7c advisory
// workflows (regression, migration-test, cross-version-smoke,
// proxied-local-smoke). All four are Bazel targets now, so nothing restores
// that namespace; a workflow naming it again would be a writer or reader with
// no counterpart.
func TestNoBlacksmithSetupGoCacheNamespace(t *testing.T) {
root := sourceRepoRoot(t)
entries, err := os.ReadDir(filepath.Join(root, ".github", "workflows"))
if err != nil {
t.Fatal(err)
}
for _, entry := range entries {
if !strings.HasSuffix(entry.Name(), ".yml") {
continue
}
if strings.Contains(readPolicyFile(t, root, ".github/workflows/"+entry.Name()), "blacksmith-sg-v1-") {
t.Errorf("%s names the retired blacksmith-sg-v1- setup-go cache namespace", entry.Name())
}
}
if _, found := readCIWorkflow(t, "main.yml").Jobs["blacksmith-setup-go-cache"]; found {
t.Error("main.yml has a blacksmith-setup-go-cache job again; it seeds a cache nothing restores")
}
}
// TestMainWorkflowHasNoSameRepoPRReachableTrigger is the direct half of F7c
// review fix S2's "pin the savers' trust boundary" ask: belt-and-suspenders
// alongside the job-level guard in TestBlacksmithSaverJobsGuardedAgainstPullRequest.
// The guard defuses the vulnerability even if one of these triggers is added
// to main.yml's `on:` block, but this test catches the trigger addition
// itself at the workflow-trust-surface level, so a future edit here gets
// flagged before anyone has to reason about whether the job-level `if:`
// still holds. (A mutation that only edits `on:`, leaving the job-level `if:`
// untouched, would otherwise not fail any other test here.)
func TestMainWorkflowHasNoSameRepoPRReachableTrigger(t *testing.T) {
type mainTriggers struct {
On map[string]any `yaml:"on"`
}
var parsed mainTriggers
text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/main.yml")
if err := yaml.Unmarshal([]byte(text), &parsed); err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"pull_request", "pull_request_target", "merge_group", "workflow_run"} {
if _, ok := parsed.On[forbidden]; ok {
t.Errorf("main.yml's on: has a %q trigger; its push-to-main cache savers are the only writers PR jobs trust, so this workflow must never become reachable from a same-repo PR/merge-queue event even with the job-level guard as a second layer", forbidden)
}
}
if _, ok := parsed.On["push"]; !ok {
t.Errorf("main.yml's on: has no push trigger: %+v", parsed.On)
}
}
// --- General sweep: no Blacksmith-reachable advisory job may save a cache --
// generalCacheSweepWorkflows is every F7c advisory workflow plus main.yml
// (the seeder). pr.yml, pr-risk.yml and bazel.yml also run jobs on
// Blacksmith, but their Go-cache topology is already pinned exhaustively,
// job-by-job and step-by-step, by TestGoCacheOwnershipTopology and
// TestBazelWorkflowCacheTopology; those jobs' runs-on expressions also
// depend on a prior job's `needs.rbe.outputs.enabled` output, which the
// minimal evalGHExpr engine below (deliberately scoped to github.*/runner.*
// context lookups) cannot resolve, so folding them into this sweep would
// either silently under-check them or require duplicating that existing
// machinery. This list is every workflow this F7c round can actually edit
// plus the one workflow (main.yml) whose seeder job this round added a new
// guard to.
var generalCacheSweepWorkflows = []string{"main.yml"}
// blacksmithTrustContexts are the two event shapes a same-repo Blacksmith
// `runs-on` ternary can route onto a `blacksmith-*` label for (F7c review fix
// S2): a trusted same-repo pull_request, and merge_group. runner.environment
// is pinned to "self-hosted" in both, matching what a job actually observes
// once it lands on a Blacksmith runner, so a step's own
// `runner.environment == 'self-hosted'`-gated logic evaluates the same way
// here as it would for real.
var blacksmithTrustContexts = map[string]map[string]string{
"pull_request": {
"github.event_name": "pull_request",
"github.event.pull_request.head.repo.full_name": "gastownhall/beads",
"github.repository": "gastownhall/beads",
"github.actor": "alice",
"github.ref": "refs/pull/1/merge",
"runner.environment": "self-hosted",
},
"merge_group": {
"github.event_name": "merge_group",
"github.repository": "gastownhall/beads",
"github.ref": "refs/heads/gh-readonly-queue/main/pr-1",
"runner.environment": "self-hosted",
},
}
// ghExprTruthyDefaultTrue evaluates expr under ctx, treating both an empty
// expr and an evaluation error as truthy/reachable: this sweep's job is to
// catch a missing or wrong save-gate, so an `if:` this minimal evaluator
// cannot parse must fail closed (assume the step/job runs) rather than
// silently skip the jobs or steps that use it.
func ghExprTruthyDefaultTrue(expr string, ctx map[string]string) bool {
if strings.TrimSpace(expr) == "" {
return true
}
v, err := evalGHExpr(expr, ctx)
if err != nil {
return true
}
return ghTruthy(v)
}
// resolveRunsOnLabel returns the runner label a job's runs-on resolves to
// under ctx: the literal string itself if it is not a `${{ ... }}`
// expression, the evaluated result if it is and evaluates to a string, or ""
// (never a Blacksmith label) if it cannot be resolved at all.
func resolveRunsOnLabel(runsOn string, ctx map[string]string) string {
if !strings.Contains(runsOn, "${{") {
return strings.TrimSpace(runsOn)
}
v, err := evalGHExpr(runsOn, ctx)
if err != nil {
return ""
}
s, ok := v.(string)
if !ok {
return ""
}
return s
}
// TestBlacksmithReachableAdvisoryJobsNeverSaveACache is the general sweep the
// F7c re-review required (S2): rather than a hand-picked table of the caches
// known about so far (B2, X1), evaluate EVERY job in EVERY generalCacheSweep-
// Workflows workflow against both blacksmithTrustContexts, and for every job
// whose runs-on actually resolves to a blacksmith-* label under one of them,
// forbid: a bare (monolithic) actions/cache step; an actions/cache/save step
// that is not gated off (would still run under) that same context; a
// setup-go step whose effective `cache` input is not disabled on a
// self-hosted runner; and a setup-node/setup-python step with any `cache`
// input set at all. This is the mechanism that would have caught N3 (docs
// setup-node cache: npm), N4 (a new bare actions/cache step in a Blacksmith
// job), and - once main.yml's seeder gained its S2 job-level guard - proves
// X1/N6 (main.yml growing a pull_request trigger) still cannot make the
// seeder's always-on save step run on a PR, without any of those needing
// their own bespoke test.
func TestBlacksmithReachableAdvisoryJobsNeverSaveACache(t *testing.T) {
// Sorted so t.Run subtest names (and therefore failure ordering) are
// stable regardless of Go's randomized map iteration.
ctxNames := make([]string, 0, len(blacksmithTrustContexts))
for name := range blacksmithTrustContexts {
ctxNames = append(ctxNames, name)
}
sort.Strings(ctxNames)
for _, file := range generalCacheSweepWorkflows {
workflow := readCIWorkflow(t, file)
for jobName, job := range workflow.Jobs {
job := job
for _, ctxName := range ctxNames {
ctx := blacksmithTrustContexts[ctxName]
t.Run(file+"/"+jobName+"/"+ctxName, func(t *testing.T) {
if !ghExprTruthyDefaultTrue(job.If, ctx) {
return // job cannot even run under this event
}
label := resolveRunsOnLabel(job.RunsOn, ctx)
if !strings.HasPrefix(label, "blacksmith-") {
return // doesn't land on Blacksmith under this context
}
// Every context under which the job is Blacksmith-reachable
// must be checked independently (not just the first one
// found): a save gate like `github.event_name !=
// 'pull_request'` is false under pull_request but true
// under merge_group, so a job reachable on Blacksmith
// under both needs both verified.
for _, step := range job.Steps {
family := actionFamily(step.Uses)
switch family {
case cacheMonolithicActionFamily:
t.Errorf("%s job %s step %q uses bare actions/cache on a Blacksmith-reachable job; it auto-saves on any key miss (B2/X1 forbid this - use actions/cache/restore + a non-PR-gated actions/cache/save)", file, jobName, step.Name)
case cacheSaveActionFamily:
if ghExprTruthyDefaultTrue(step.If, ctx) {
t.Errorf("%s job %s step %q (actions/cache/save) has if=%q, which still runs under a same-repo Blacksmith %s event; it must be gated off", file, jobName, step.Name, step.If, ctxName)
}
case setupGoActionFamily:
cacheVal, ok := step.With["cache"]
wouldCache := true
if ok {
if !strings.Contains(cacheVal, "${{") {
wouldCache = cacheVal == "true"
} else if v, err := evalGHExpr(cacheVal, ctx); err == nil {
wouldCache = ghTruthy(v)
}
}
if wouldCache {
t.Errorf("%s job %s setup-go step %q has cache=%q, which stays enabled on a self-hosted (Blacksmith) runner; it must disable its own implicit cache there", file, jobName, step.Name, cacheVal)
}
case setupNodeActionFamily, setupPythonActionFamily:
if cacheVal, ok := step.With["cache"]; ok && cacheVal != "" {
t.Errorf("%s job %s step %q sets cache=%q on a Blacksmith-reachable job; setup-node/setup-python's own cache uses a bare actions/cache internally", file, jobName, step.Name, cacheVal)
}
}
}
})
}
}
}
}