From c86a87bb2071462a0996fecabd0b5cbb1da9c3c2 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 19 Oct 2026 22:02:44 +0000 Subject: [PATCH] test(oracle-a): pin BEADS_TIP_SEED so human-output scenarios stop flaking (#7707) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a successful human-output create/show/list/ready, bd may append a "💡 Tip:" line chosen by a probability roll whose RNG is clock-seeded unless BEADS_TIP_SEED is set. run_scenario env_clear()s and passes only PATH/HOME/TMPDIR + BEADS_TEST_MODE=1, so on a host with ~/.claude (claude_setup tip eligible, p=0.6) every bd process rolled independently and out-of-scope human-output scenarios (create_human_readable_output, show_short_mode_text, show_multiple_short, ready_empty_no_issues_plain, ...) diverged main-vs-main. Pass BEADS_TIP_SEED=1214 to every step for both reference and candidate. Go math/rand's first Float64() for that seed is 0.0000159, so the highest-priority eligible tip always wins its roll. Where the host makes a tip eligible, it stays visible and lands on a deterministic step, so a change to which tip prints, on which step, or to its text (including a tip leaking into --json) still diverges instead of being normalized away. Where no tip is eligible (no ~/.claude, beads plugin enabled, or a `bd prime` hook) none prints and tip changes go unobserved; the README now says so. The new unit test spawns printenv through run_scenario. That races the existing stub-bd test, which writes its stub with std::fs::write and then execs it: a fork while the write fd is open makes the exec fail with ETXTBSY (rust-lang/rust#114554). Serialize both spawning tests on a SPAWN mutex. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-authored-by: Claude Opus 5.5 (1M context) --- tests/oracle-a/README.md | 19 +++++++-- tests/oracle-a/harness/PROVENANCE.md | 12 ++++-- tests/oracle-a/harness/src/differential.rs | 45 +++++++++++++++++++++- 3 files changed, 66 insertions(+), 10 deletions(-) diff --git a/tests/oracle-a/README.md b/tests/oracle-a/README.md index dce5c604f..5728755a9 100644 --- a/tests/oracle-a/README.md +++ b/tests/oracle-a/README.md @@ -137,10 +137,21 @@ the loop — every scenario uses embedded Dolt in its own tempdir. gate. No scenario sets one today; use another key name in any that would. - **The scenario environment is scrubbed, and `BEADS_TEST_MODE=1` is ungated.** Scenarios run with an explicit env whitelist (`PATH`/`HOME`/`TMPDIR` + - `BEADS_TEST_MODE=1`) rather than the inherited host env, so a green certifies the - *same* configuration every run. But `BEADS_TEST_MODE=1` itself changes store - construction (auto-server decisions off, port rewiring), so this rig does NOT - cover the mode/topology construction paths — do not cite Oracle A green for those. + `BEADS_TEST_MODE=1` + `BEADS_TIP_SEED`) rather than the inherited host env, so a + green certifies the *same* configuration every run. But `BEADS_TEST_MODE=1` + itself changes store construction (auto-server decisions off, port rewiring), so + this rig does NOT cover the mode/topology construction paths — do not cite Oracle + A green for those. +- **Tips are pinned, not erased, and depend on the host.** Human output may end + with a `💡 Tip:` line picked by a probability roll. `BEADS_TIP_SEED` (`TIP_SEED` + in `harness/src/differential.rs`) pins that roll so the highest-priority eligible + tip always wins: its text, the step it prints on, and its frequency gate are + compared like any other output, but a tip's probability value is not gated. + Whether the `claude_setup` tip is eligible depends on the host `HOME` + (`~/.claude` present, no beads plugin enabled, no `bd prime` hook). Both binaries + run on the same host, so this is symmetric, but on a host where no tip is + eligible (a CI runner, or a beads developer with the plugin installed) none + prints and tip changes go unobserved, including a tip leaking into `--json`. - **This is a same-backend gate.** Oracle A is same-backend (embedded Dolt) before-vs-after. It is a refactor-safety net, not a cross-backend conformance run. diff --git a/tests/oracle-a/harness/PROVENANCE.md b/tests/oracle-a/harness/PROVENANCE.md index 133921641..40c21b3da 100644 --- a/tests/oracle-a/harness/PROVENANCE.md +++ b/tests/oracle-a/harness/PROVENANCE.md @@ -31,10 +31,14 @@ the current "before" rather than a checked-in snapshot. ## Environment contract The scenario runner scrubs the environment to an explicit whitelist -(`PATH`/`HOME`/`TMPDIR` + `BEADS_TEST_MODE=1`) via `env_clear()` rather than -inheriting the host environment, so a green certifies the same configuration on -every host and run. The `BEADS_TEST_MODE=1` store-construction delta is a -documented ungated gap (see `../README.md`). +(`PATH`/`HOME`/`TMPDIR` + `BEADS_TEST_MODE=1` + `BEADS_TIP_SEED`) via +`env_clear()` rather than inheriting the host environment, so a green certifies +the same configuration on every host and run. `BEADS_TIP_SEED` (`TIP_SEED` in +`src/differential.rs`) pins bd's otherwise clock-seeded tip roll, so where the +host makes a tip eligible it prints on the same step every run and tip changes +still diverge; where no tip is eligible none prints (see `../README.md`). The +`BEADS_TEST_MODE=1` store-construction delta is a documented ungated gap (see +`../README.md`). ## Cargo manifest diff --git a/tests/oracle-a/harness/src/differential.rs b/tests/oracle-a/harness/src/differential.rs index 606fa2a2f..cfa3464c0 100644 --- a/tests/oracle-a/harness/src/differential.rs +++ b/tests/oracle-a/harness/src/differential.rs @@ -20,6 +20,19 @@ use std::sync::OnceLock; pub const ACTOR_NAME: &str = "CI Bot"; pub const ACTOR_EMAIL: &str = "ci@beads.test"; +/// The `BEADS_TIP_SEED` every step runs with. After a successful human-output +/// `create`/`show`/`list`/`ready`, bd may append a "💡 Tip:" line chosen by a +/// probability roll, and unless BEADS_TIP_SEED is set that RNG is seeded from +/// the clock (cmd/bd/tips.go `initTipRand`), so the same binary prints the tip +/// on a different step, or not at all, from run to run. For this seed Go's +/// `math/rand` returns 0.0000159 as its first `Float64()`, so the +/// highest-priority eligible tip always wins its roll. Which tip prints, and on +/// which step, then depends only on eligibility, priority and the tip's +/// frequency gate, so any change to which tip prints, on which step, or to its +/// text still diverges instead of being normalized away. (A tip only prints +/// where the host makes it eligible; see tests/oracle-a/README.md.) +pub const TIP_SEED: &str = "1214"; + /// A scenario: a prefix to `bd init` with, and an ordered list of argv steps /// (each run after init in the same workspace). A scenario whose first step is /// its own `init` owns initialization: it starts in an empty workspace and @@ -222,12 +235,14 @@ pub fn run_scenario(bin: &Path, sc: &Scenario) -> std::io::Result { // run and on any host. (BEADS_TEST_MODE=1 is retained deliberately; its // store-construction delta — auto-server decisions off, port rewiring — // is a documented ungated gap: this rig does not cover mode/topology - // construction paths. See tests/oracle-a/README.md.) + // construction paths. See tests/oracle-a/README.md.) BEADS_TIP_SEED pins + // bd's tip roll; see TIP_SEED. let mut cmd = Command::new(bin); cmd.args(args) .current_dir(work.path()) .env_clear() - .env("BEADS_TEST_MODE", "1"); + .env("BEADS_TEST_MODE", "1") + .env("BEADS_TIP_SEED", TIP_SEED); for key in ["PATH", "HOME", "TMPDIR"] { if let Ok(val) = std::env::var(key) { cmd.env(key, val); @@ -420,6 +435,13 @@ pub fn diff(reference: &Trace, candidate: &Trace, ordered: bool) -> Vec = std::sync::Mutex::new(()); + #[test] fn ordered_comparison_bites_on_permutations_multiset_does_not() { // A reordered array is EQUAL as a multiset but UNEQUAL as a sequence — this is @@ -538,6 +560,7 @@ mod tests { #[cfg(unix)] #[test] fn run_scenario_collapses_the_workspace_in_both_step_streams() { + let _spawn = SPAWN.lock().unwrap_or_else(|e| e.into_inner()); // A stub bd that echoes its cwd on stdout and its symlink-resolved form // on stderr, like the "Found existing Dolt database" error: both must // reach the trace as , so the run's random tempdir never does. @@ -660,4 +683,22 @@ mod tests { // and multiset cardinality IS caught assert!(!stdout_equiv(r#"[{"id":"a"},{"id":"a"}]"#, r#"[{"id":"a"}]"#, false)); } + + #[cfg(unix)] + #[test] + fn run_scenario_pins_the_tip_seed_for_every_step() { + let _spawn = SPAWN.lock().unwrap_or_else(|e| e.into_inner()); + // printenv stands in for bd: each step prints BEADS_TIP_SEED from the + // environment the harness hands every bd step (the harness's own + // `init -p t --quiet` just exits non-zero, which run_scenario ignores). + // Unset, real bd seeds its tip roll from the clock, so the same binary + // prints a "💡 Tip:" line on some runs and not on others. + let sc = Scenario::new("tip_seed", "t", &[&["BEADS_TIP_SEED"], &["BEADS_TIP_SEED"]]); + let trace = run_scenario(Path::new("printenv"), &sc).expect("run printenv"); + assert_eq!(trace.steps.len(), 2); + let want = format!("{TIP_SEED}\n"); + for step in &trace.steps { + assert_eq!((step.exit, step.stdout.as_str()), (0, want.as_str())); + } + } }