diff --git a/scripts/README.md b/scripts/README.md index 6f7a0aa9e..23c087689 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -66,6 +66,16 @@ signals such as `.beads/` changes or missing tests, and the required contributor-protection next steps. It does not replace code review or local validation. +Base-branch health is supplemented by a warn-only PR-gate sample: the last 60 +completed `pull_request`-event runs, grouped per workflow. A single workflow +whose decisive runs (>= 5, across >= 3 distinct head branches) are all +failure-class is reported as a broken PR gate — the case where a job that +exists only in the PR workflow is red for every PR while the base branch shows +green. This detector never blocks (deliberately: automation classifies +unrecognized `[block]` lines as genuine merge blockers and would park merge +lanes on a false positive); it is skipped entirely while the base branch is +red. + ## gh-body-lint Lint Markdown files before posting them with `gh ... --body-file`. diff --git a/scripts/pr-preflight.sh b/scripts/pr-preflight.sh index 64cf16368..677d30de9 100755 --- a/scripts/pr-preflight.sh +++ b/scripts/pr-preflight.sh @@ -23,6 +23,8 @@ What this checks: - draft, review, mergeability, and check status - CI health of the base branch (red base makes "failures are pre-existing" reasoning unsafe; set PR_PREFLIGHT_BLOCK_RED_BASE=1 to block instead of warn) + - a supplemental, warn-only sample for a PR-gate workflow (e.g. pr.yml) that + is red for every PR while the base branch itself looks green - risky diff signals: .beads data, missing tests for code changes, large diffs - contributor-protection next steps and attribution reminders @@ -292,6 +294,59 @@ else warn "Could not determine CI health of base branch ${base_ref}; check it manually before merging." fi +# PR-gate health sample. The base-branch check above reads runs on the base +# branch itself, so it cannot see a job that exists only in the PR workflow +# (pr.yml) - such a job never runs on the base branch, and one broken job +# there is diluted to invisibility by every other green PR workflow if runs +# are judged in aggregate. Sample recent completed pull_request-event runs +# across PRs, group by workflow, and flag any single workflow whose decisive +# runs are uniformly failing across several distinct branches. +# +# This is a supplemental, warn-only detector, deliberately never coupled to +# PR_PREFLIGHT_BLOCK_RED_BASE: a [block] line here matches neither the +# red-base nor the transient-block patterns pr-babysit's patrol looks for, +# so blocking would park every merge lane and silently revoke the red-base +# merge exception. (Wiring this signal into the patrol is a separate, +# coordinated change.) Skip the sample entirely when the base branch is +# already red - that already explains per-PR redness, and avoids reporting +# "the base branch shows green" right after a red-base line above - and stay +# silent on an empty, unreadable, or mixed-health sample. +if [[ "$base_red_count" -eq 0 && "$base_green_count" -gt 0 ]]; then + # Only diagnose the PR gate against a base branch known to be green: on a red + # base the red-base message already explains PR redness, and on an + # undetermined base the "while the base branch shows green" explanation + # would contradict the warn printed just above. + pr_gate_runs=$(gh run list --repo "$repo" --event pull_request --status completed \ + --limit 60 --json conclusion,headBranch,workflowName,workflowDatabaseId,createdAt,url 2>/dev/null) || pr_gate_runs="" + # gh can exit 0 with non-JSON (or empty) output; treat anything that is not + # a JSON array the same as an empty sample rather than let jq abort under -e. + jq -e 'type == "array"' >/dev/null 2>&1 <<<"${pr_gate_runs:-}" || pr_gate_runs='[]' + # Grouped by workflowDatabaseId (stable identity), not display name: two + # workflow files may share a legal name:, and org/ruleset runs can have no + # name at all. Known limitation, accepted for a warn-only supplemental + # detector: the sample is repo-wide, so runs from PRs targeting other base + # branches are mixed in — resolving each run's PR to filter by base would + # cost one API call per run. + pr_gate_broken=$(jq ' + [group_by(.workflowDatabaseId)[] + | { workflow: (.[0].workflowName // "unknown"), + decisive: [.[] | select( + (.conclusion // "") == "success" or + ((.conclusion // "") | test("^(failure|timed_out|action_required|startup_failure)$")) + )] } + | select((.decisive | length) >= 5) + | select((.decisive | map(.headBranch) | unique | length) >= 3) + | select(all(.decisive[]; (.conclusion // "") | test("^(failure|timed_out|action_required|startup_failure)$"))) + ] | .[0] // empty' <<<"$pr_gate_runs") || pr_gate_broken="" + if [[ -n "$pr_gate_broken" ]]; then + pr_gate_workflow=$(jq -r '.workflow' <<<"$pr_gate_broken") + pr_gate_count=$(jq -r '.decisive | length' <<<"$pr_gate_broken") + pr_gate_heads=$(jq -r '.decisive | map(.headBranch) | unique | length' <<<"$pr_gate_broken") + pr_gate_newest=$(jq -r '.decisive | sort_by(.createdAt) | last | "\(.createdAt) \(.url)"' <<<"$pr_gate_broken") + warn "PR gate workflow '${pr_gate_workflow}' appears broken: all ${pr_gate_count} of its recent completed pull_request runs across ${pr_gate_heads} branches failed (newest: ${pr_gate_newest}). A job that exists only in that workflow may be red for every PR while the base branch shows green. Investigate before trusting per-PR check verdicts." + fi +fi + # statusCheckRollup keeps every check run on the head SHA, including runs # from cancelled or superseded check suites. A head whose earlier suite was # cancelled mid-flight carries stale CANCELLED/FAILURE entries forever, next diff --git a/scripts/pr_preflight_test.go b/scripts/pr_preflight_test.go index 39e75e944..e06d6f1aa 100644 --- a/scripts/pr_preflight_test.go +++ b/scripts/pr_preflight_test.go @@ -44,7 +44,19 @@ const ( printf '%s\n' "{\"number\":${PR_NUMBER},\"title\":\"Compatibility fixture\",\"author\":{\"login\":\"contributor\"},\"url\":\"${PR_URL}\",\"baseRefName\":\"main\",\"headRefName\":\"fix/preflight\",\"headRepositoryOwner\":{\"login\":\"contributor\"},\"isCrossRepository\":true,\"isDraft\":false,\"maintainerCanModify\":true,\"mergeStateStatus\":\"CLEAN\",\"mergeable\":\"MERGEABLE\",\"reviewDecision\":\"APPROVED\",\"changedFiles\":1,\"additions\":1,\"deletions\":0,\"files\":[{\"path\":\"README.md\"}],\"statusCheckRollup\":${PR_ROLLUP:-[]},\"latestReviews\":[]}" ;; "run list") - printf '[]\n' + is_pr_gate_call=0 + prev="" + for arg in "$@"; do + if [ "$prev" = "--event" ] && [ "$arg" = "pull_request" ]; then + is_pr_gate_call=1 + fi + prev="$arg" + done + if [ "$is_pr_gate_call" = 1 ]; then + printf '%s\n' "${PR_GATE_RUN_LIST:-[]}" + else + printf '%s\n' "${BASE_RUN_LIST:-[]}" + fi ;; "api graphql") if [ "$GRAPHQL_EXIT" != 0 ]; then @@ -69,6 +81,9 @@ type preflightFixture struct { graphQLResponse string graphQLExit string rollup string + baseRunList string + prGateRunList string + blockRedBase string } type preflightRun struct { @@ -332,6 +347,173 @@ func TestPRPreflightChecksUseLatestRunPerName(t *testing.T) { }) } +// The base-branch health check above reads gh run list --branch , which +// only sees workflows that actually run on the base branch. A job that exists +// only in the PR workflow (pr.yml) never runs there, and even sampling +// pull_request-event runs in aggregate would dilute one broken workflow's +// signal against every other green PR workflow. This supplemental, warn-only +// detector groups the sample by workflow and flags any single workflow that +// is uniformly failing across several distinct branches. +func TestPRPreflightPRGateHealthSample(t *testing.T) { + // The detector only speaks against a base branch known to be green; every + // case that exercises it therefore supplies a green base sample. + greenBase := `[{"conclusion":"success","workflowName":"CI","createdAt":"2026-07-28T12:00:00Z","url":"https://github.com/o/r/actions/base-0"}]` + sixRunsThreeHeadsAllFailed := `[` + + `{"conclusion":"failure","headBranch":"branch-a","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T10:00:00Z","url":"https://github.com/o/r/actions/runs/1"},` + + `{"conclusion":"failure","headBranch":"branch-a","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T09:00:00Z","url":"https://github.com/o/r/actions/runs/2"},` + + `{"conclusion":"failure","headBranch":"branch-b","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T08:00:00Z","url":"https://github.com/o/r/actions/runs/3"},` + + `{"conclusion":"failure","headBranch":"branch-b","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T07:00:00Z","url":"https://github.com/o/r/actions/runs/4"},` + + `{"conclusion":"failure","headBranch":"branch-c","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T06:00:00Z","url":"https://github.com/o/r/actions/runs/5"},` + + `{"conclusion":"failure","headBranch":"branch-c","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T05:00:00Z","url":"https://github.com/o/r/actions/runs/6"}` + + `]` + + t.Run("all failed across 3+ heads with 5+ decisive runs warns and names the workflow", func(t *testing.T) { + run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: sixRunsThreeHeadsAllFailed}) + if run.err != nil { + t.Fatalf("expected warn-only exit, got error: %v\n%s", run.err, run.output) + } + want := "[warn] PR gate workflow 'PR CI' appears broken: all 6 of its recent completed pull_request runs across 3 branches failed (newest: 2026-07-28T10:00:00Z https://github.com/o/r/actions/runs/1)." + if !strings.Contains(run.output, want) { + t.Fatalf("expected broken-PR-gate warning naming the workflow and newest run:\nwant substring: %s\ngot:\n%s", want, run.output) + } + }) + + t.Run("stays warn-only even under PR_PREFLIGHT_BLOCK_RED_BASE=1", func(t *testing.T) { + // The warn-only property is the load-bearing one: mybd's pr-babysit + // classifies any unrecognized [block] line as a genuine merge blocker + // and parks merge lanes on it, and it withholds the base-fix + // exception whenever a non-red-base block is present. This detector + // must therefore never escalate to a block, regardless of the + // red-base escalation env var. + run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: sixRunsThreeHeadsAllFailed, blockRedBase: "1"}) + if run.err != nil { + t.Fatalf("expected warn-only exit even with PR_PREFLIGHT_BLOCK_RED_BASE=1, got error: %v\n%s", run.err, run.output) + } + if !strings.Contains(run.output, "[warn] PR gate workflow 'PR CI' appears broken") { + t.Fatalf("expected the broken-PR-gate warn to still fire:\n%s", run.output) + } + if strings.Contains(run.output, "[block] PR gate") { + t.Fatalf("PR-gate detector must never emit a [block] line:\n%s", run.output) + } + }) + + t.Run("one success in the workflow's sample stays silent", func(t *testing.T) { + mostlyFailedOneSuccess := strings.Replace(sixRunsThreeHeadsAllFailed, `"conclusion":"failure","headBranch":"branch-c","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T05:00:00Z"`, `"conclusion":"success","headBranch":"branch-c","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T05:00:00Z"`, 1) + run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: mostlyFailedOneSuccess}) + if run.err != nil { + t.Fatalf("expected exit 0, got error: %v\n%s", run.err, run.output) + } + if strings.Contains(run.output, "PR gate workflow") { + t.Fatalf("a workflow sample with a success should stay silent:\n%s", run.output) + } + }) + + t.Run("all failed but only 2 distinct heads stays silent", func(t *testing.T) { + twoHeadsAllFailed := `[` + + `{"conclusion":"failure","headBranch":"branch-a","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T10:00:00Z","url":"https://github.com/o/r/actions/runs/1"},` + + `{"conclusion":"failure","headBranch":"branch-a","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T09:00:00Z","url":"https://github.com/o/r/actions/runs/2"},` + + `{"conclusion":"failure","headBranch":"branch-a","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T08:00:00Z","url":"https://github.com/o/r/actions/runs/3"},` + + `{"conclusion":"failure","headBranch":"branch-b","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T07:00:00Z","url":"https://github.com/o/r/actions/runs/4"},` + + `{"conclusion":"failure","headBranch":"branch-b","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T06:00:00Z","url":"https://github.com/o/r/actions/runs/5"},` + + `{"conclusion":"failure","headBranch":"branch-b","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T05:00:00Z","url":"https://github.com/o/r/actions/runs/6"}` + + `]` + run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: twoHeadsAllFailed}) + if run.err != nil { + t.Fatalf("expected exit 0, got error: %v\n%s", run.err, run.output) + } + if strings.Contains(run.output, "PR gate workflow") { + t.Fatalf("only 2 distinct heads should stay silent:\n%s", run.output) + } + }) + + t.Run("empty sample stays silent", func(t *testing.T) { + run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: "[]"}) + if run.err != nil { + t.Fatalf("expected exit 0, got error: %v\n%s", run.err, run.output) + } + if strings.Contains(run.output, "PR gate workflow") { + t.Fatalf("empty sample should stay silent:\n%s", run.output) + } + }) + + t.Run("mixed workflows: only the all-red workflow fires, and it is named", func(t *testing.T) { + mixedWorkflows := `[` + + `{"conclusion":"success","headBranch":"branch-a","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T12:00:00Z","url":"https://github.com/o/r/actions/runs/201"},` + + `{"conclusion":"success","headBranch":"branch-b","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T11:30:00Z","url":"https://github.com/o/r/actions/runs/202"},` + + `{"conclusion":"failure","headBranch":"branch-a","workflowName":"Contract corpus","workflowDatabaseId":202,"createdAt":"2026-07-28T10:00:00Z","url":"https://github.com/o/r/actions/runs/101"},` + + `{"conclusion":"failure","headBranch":"branch-a","workflowName":"Contract corpus","workflowDatabaseId":202,"createdAt":"2026-07-28T09:00:00Z","url":"https://github.com/o/r/actions/runs/102"},` + + `{"conclusion":"failure","headBranch":"branch-b","workflowName":"Contract corpus","workflowDatabaseId":202,"createdAt":"2026-07-28T08:00:00Z","url":"https://github.com/o/r/actions/runs/103"},` + + `{"conclusion":"failure","headBranch":"branch-b","workflowName":"Contract corpus","workflowDatabaseId":202,"createdAt":"2026-07-28T07:00:00Z","url":"https://github.com/o/r/actions/runs/104"},` + + `{"conclusion":"failure","headBranch":"branch-c","workflowName":"Contract corpus","workflowDatabaseId":202,"createdAt":"2026-07-28T06:00:00Z","url":"https://github.com/o/r/actions/runs/105"}` + + `]` + run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: mixedWorkflows}) + if run.err != nil { + t.Fatalf("expected exit 0, got error: %v\n%s", run.err, run.output) + } + want := "[warn] PR gate workflow 'Contract corpus' appears broken: all 5 of its recent completed pull_request runs across 3 branches failed (newest: 2026-07-28T10:00:00Z https://github.com/o/r/actions/runs/101)." + if !strings.Contains(run.output, want) { + t.Fatalf("expected the all-red workflow to be named despite a healthy sibling workflow:\nwant substring: %s\ngot:\n%s", want, run.output) + } + if strings.Contains(run.output, "'PR CI'") { + t.Fatalf("the healthy workflow must not be named as broken:\n%s", run.output) + } + }) + + t.Run("undetermined base health skips the sample", func(t *testing.T) { + // With no decisive base runs at all, preflight has just warned that it + // could not determine base health; diagnosing "red for every PR while + // the base branch shows green" would contradict that line. + run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: "[]", prGateRunList: sixRunsThreeHeadsAllFailed}) + if run.err != nil { + t.Fatalf("expected exit 0, got error: %v\n%s", run.err, run.output) + } + if strings.Contains(run.output, "PR gate workflow") { + t.Fatalf("the PR-gate sample must be skipped when base health is undetermined:\n%s", run.output) + } + }) + + t.Run("red base skips the sample even when a workflow would otherwise fire", func(t *testing.T) { + redBaseRunList := `[{"conclusion":"failure","workflowName":"CI","createdAt":"2026-07-28T09:00:00Z","url":"https://github.com/o/r/actions/base-1"}]` + run := runPRPreflightWithFakeGH(t, preflightFixture{ + baseRunList: redBaseRunList, + prGateRunList: sixRunsThreeHeadsAllFailed, + }) + if run.err != nil { + t.Fatalf("expected exit 0 (warn-only), got error: %v\n%s", run.err, run.output) + } + if !strings.Contains(run.output, "Base branch main CI is RED") { + t.Fatalf("expected the existing red-base warning to still fire:\n%s", run.output) + } + if strings.Contains(run.output, "PR gate workflow") { + t.Fatalf("the PR-gate sample must be skipped while the base is red:\n%s", run.output) + } + }) + + t.Run("unreadable/non-JSON sample stays silent", func(t *testing.T) { + run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: "gh: rate limit exceeded"}) + if run.err != nil { + t.Fatalf("expected exit 0, got error: %v\n%s", run.err, run.output) + } + if strings.Contains(run.output, "PR gate workflow") { + t.Fatalf("an unreadable sample should stay silent, not error:\n%s", run.output) + } + }) + + t.Run("BASE_RUN_LIST positively drives the base-branch dispatch, not the PR-gate one", func(t *testing.T) { + greenBaseRunList := `[{"conclusion":"success","workflowName":"CI","createdAt":"2026-07-28T12:00:00Z","url":"https://github.com/o/r/actions/base-2"}]` + run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBaseRunList}) + if run.err != nil { + t.Fatalf("expected exit 0, got error: %v\n%s", run.err, run.output) + } + if !strings.Contains(run.output, "[pass] Base branch main CI is green (latest completed run of each of 1 workflow(s) succeeded).") { + t.Fatalf("expected BASE_RUN_LIST to reach the base-branch call, not the PR-gate one:\n%s", run.output) + } + if strings.Contains(run.output, "PR gate workflow") { + t.Fatalf("default empty PR-gate sample should stay silent:\n%s", run.output) + } + }) +} + func runPRPreflightWithFakeGH(t *testing.T, fixture preflightFixture) preflightRun { t.Helper() @@ -353,6 +535,15 @@ func runPRPreflightWithFakeGH(t *testing.T, fixture preflightFixture) preflightR if fixture.rollup == "" { fixture.rollup = "[]" } + if fixture.baseRunList == "" { + fixture.baseRunList = "[]" + } + if fixture.prGateRunList == "" { + fixture.prGateRunList = "[]" + } + if fixture.blockRedBase == "" { + fixture.blockRedBase = "0" + } bash, path := prPreflightProcessTools(t) @@ -395,6 +586,9 @@ func runPRPreflightWithFakeGH(t *testing.T, fixture preflightFixture) preflightR "PR_NUMBER="+fixture.returnedNumber, "PR_URL="+fixture.canonicalURL, "PR_ROLLUP="+fixture.rollup, + "BASE_RUN_LIST="+fixture.baseRunList, + "PR_GATE_RUN_LIST="+fixture.prGateRunList, + "PR_PREFLIGHT_BLOCK_RED_BASE="+fixture.blockRedBase, ) out, runErr := cmd.CombinedOutput() logBytes, readErr := os.ReadFile(callLog)