diff --git a/tests/oracle-a/README.md b/tests/oracle-a/README.md index dce5c604f..5728755a9 100644 --- a/tests/oracle-a/README.md +++ b/tests/oracle-a/README.md @@ -137,10 +137,21 @@ the loop — every scenario uses embedded Dolt in its own tempdir. gate. No scenario sets one today; use another key name in any that would. - **The scenario environment is scrubbed, and `BEADS_TEST_MODE=1` is ungated.** Scenarios run with an explicit env whitelist (`PATH`/`HOME`/`TMPDIR` + - `BEADS_TEST_MODE=1`) rather than the inherited host env, so a green certifies the - *same* configuration every run. But `BEADS_TEST_MODE=1` itself changes store - construction (auto-server decisions off, port rewiring), so this rig does NOT - cover the mode/topology construction paths — do not cite Oracle A green for those. + `BEADS_TEST_MODE=1` + `BEADS_TIP_SEED`) rather than the inherited host env, so a + green certifies the *same* configuration every run. But `BEADS_TEST_MODE=1` + itself changes store construction (auto-server decisions off, port rewiring), so + this rig does NOT cover the mode/topology construction paths — do not cite Oracle + A green for those. +- **Tips are pinned, not erased, and depend on the host.** Human output may end + with a `💡 Tip:` line picked by a probability roll. `BEADS_TIP_SEED` (`TIP_SEED` + in `harness/src/differential.rs`) pins that roll so the highest-priority eligible + tip always wins: its text, the step it prints on, and its frequency gate are + compared like any other output, but a tip's probability value is not gated. + Whether the `claude_setup` tip is eligible depends on the host `HOME` + (`~/.claude` present, no beads plugin enabled, no `bd prime` hook). Both binaries + run on the same host, so this is symmetric, but on a host where no tip is + eligible (a CI runner, or a beads developer with the plugin installed) none + prints and tip changes go unobserved, including a tip leaking into `--json`. - **This is a same-backend gate.** Oracle A is same-backend (embedded Dolt) before-vs-after. It is a refactor-safety net, not a cross-backend conformance run. diff --git a/tests/oracle-a/harness/PROVENANCE.md b/tests/oracle-a/harness/PROVENANCE.md index 133921641..40c21b3da 100644 --- a/tests/oracle-a/harness/PROVENANCE.md +++ b/tests/oracle-a/harness/PROVENANCE.md @@ -31,10 +31,14 @@ the current "before" rather than a checked-in snapshot. ## Environment contract The scenario runner scrubs the environment to an explicit whitelist -(`PATH`/`HOME`/`TMPDIR` + `BEADS_TEST_MODE=1`) via `env_clear()` rather than -inheriting the host environment, so a green certifies the same configuration on -every host and run. The `BEADS_TEST_MODE=1` store-construction delta is a -documented ungated gap (see `../README.md`). +(`PATH`/`HOME`/`TMPDIR` + `BEADS_TEST_MODE=1` + `BEADS_TIP_SEED`) via +`env_clear()` rather than inheriting the host environment, so a green certifies +the same configuration on every host and run. `BEADS_TIP_SEED` (`TIP_SEED` in +`src/differential.rs`) pins bd's otherwise clock-seeded tip roll, so where the +host makes a tip eligible it prints on the same step every run and tip changes +still diverge; where no tip is eligible none prints (see `../README.md`). The +`BEADS_TEST_MODE=1` store-construction delta is a documented ungated gap (see +`../README.md`). ## Cargo manifest diff --git a/tests/oracle-a/harness/src/differential.rs b/tests/oracle-a/harness/src/differential.rs index 606fa2a2f..cfa3464c0 100644 --- a/tests/oracle-a/harness/src/differential.rs +++ b/tests/oracle-a/harness/src/differential.rs @@ -20,6 +20,19 @@ use std::sync::OnceLock; pub const ACTOR_NAME: &str = "CI Bot"; pub const ACTOR_EMAIL: &str = "ci@beads.test"; +/// The `BEADS_TIP_SEED` every step runs with. After a successful human-output +/// `create`/`show`/`list`/`ready`, bd may append a "💡 Tip:" line chosen by a +/// probability roll, and unless BEADS_TIP_SEED is set that RNG is seeded from +/// the clock (cmd/bd/tips.go `initTipRand`), so the same binary prints the tip +/// on a different step, or not at all, from run to run. For this seed Go's +/// `math/rand` returns 0.0000159 as its first `Float64()`, so the +/// highest-priority eligible tip always wins its roll. Which tip prints, and on +/// which step, then depends only on eligibility, priority and the tip's +/// frequency gate, so any change to which tip prints, on which step, or to its +/// text still diverges instead of being normalized away. (A tip only prints +/// where the host makes it eligible; see tests/oracle-a/README.md.) +pub const TIP_SEED: &str = "1214"; + /// A scenario: a prefix to `bd init` with, and an ordered list of argv steps /// (each run after init in the same workspace). A scenario whose first step is /// its own `init` owns initialization: it starts in an empty workspace and @@ -222,12 +235,14 @@ pub fn run_scenario(bin: &Path, sc: &Scenario) -> std::io::Result { // run and on any host. (BEADS_TEST_MODE=1 is retained deliberately; its // store-construction delta — auto-server decisions off, port rewiring — // is a documented ungated gap: this rig does not cover mode/topology - // construction paths. See tests/oracle-a/README.md.) + // construction paths. See tests/oracle-a/README.md.) BEADS_TIP_SEED pins + // bd's tip roll; see TIP_SEED. let mut cmd = Command::new(bin); cmd.args(args) .current_dir(work.path()) .env_clear() - .env("BEADS_TEST_MODE", "1"); + .env("BEADS_TEST_MODE", "1") + .env("BEADS_TIP_SEED", TIP_SEED); for key in ["PATH", "HOME", "TMPDIR"] { if let Ok(val) = std::env::var(key) { cmd.env(key, val); @@ -420,6 +435,13 @@ pub fn diff(reference: &Trace, candidate: &Trace, ordered: bool) -> Vec = std::sync::Mutex::new(()); + #[test] fn ordered_comparison_bites_on_permutations_multiset_does_not() { // A reordered array is EQUAL as a multiset but UNEQUAL as a sequence — this is @@ -538,6 +560,7 @@ mod tests { #[cfg(unix)] #[test] fn run_scenario_collapses_the_workspace_in_both_step_streams() { + let _spawn = SPAWN.lock().unwrap_or_else(|e| e.into_inner()); // A stub bd that echoes its cwd on stdout and its symlink-resolved form // on stderr, like the "Found existing Dolt database" error: both must // reach the trace as , so the run's random tempdir never does. @@ -660,4 +683,22 @@ mod tests { // and multiset cardinality IS caught assert!(!stdout_equiv(r#"[{"id":"a"},{"id":"a"}]"#, r#"[{"id":"a"}]"#, false)); } + + #[cfg(unix)] + #[test] + fn run_scenario_pins_the_tip_seed_for_every_step() { + let _spawn = SPAWN.lock().unwrap_or_else(|e| e.into_inner()); + // printenv stands in for bd: each step prints BEADS_TIP_SEED from the + // environment the harness hands every bd step (the harness's own + // `init -p t --quiet` just exits non-zero, which run_scenario ignores). + // Unset, real bd seeds its tip roll from the clock, so the same binary + // prints a "💡 Tip:" line on some runs and not on others. + let sc = Scenario::new("tip_seed", "t", &[&["BEADS_TIP_SEED"], &["BEADS_TIP_SEED"]]); + let trace = run_scenario(Path::new("printenv"), &sc).expect("run printenv"); + assert_eq!(trace.steps.len(), 2); + let want = format!("{TIP_SEED}\n"); + for step in &trace.steps { + assert_eq!((step.exit, step.stdout.as_str()), (0, want.as_str())); + } + } }