fix: make posterior_of reproducible across processes
`ResolvedTerms::unseen` was a `HashMap<String, f64>` and three float reductions iterated it. Addition is not associative and Rust seeds its default hasher per process, so `posterior_of` returned different bits run to run on identical input: measured over 40 processes, two distinct sigma bit patterns, and five distinct values from `expected_variance_reduction` spanning about 7 ULP. A `BTreeMap` fixes it by construction. 40/40 identical after, 24/16 before. The cross-batch conflict scan had the same cause with a different symptom. It returns on the FIRST conflict, so hash order decided WHICH competitor the error blamed — 15 different competitors named across 40 runs on identical input. The error fired every time; only its content was a lottery, which sends a reader after the wrong key. Now scanned in sorted order. Magnitude was 1-7 ULP throughout, so no decision changes. The cost was reproducibility: a golden test over these would flake at a low rate, which is the worst kind of CI failure to diagnose. tests/cross_process_determinism.rs re-executes the test binary and compares bits, because an in-process test CANNOT see this — every sample in one process shares one hasher seed. That is not hypothetical: tests/determinism.rs compares four thread counts inside one process and passed throughout while this was live. Tuning that fixture took a measurement. Coefficients spread over nine decades detected the bug in roughly one run in forty, because the small terms fall below the running total's ULP and are absorbed whatever the order. Comparable magnitudes keep every term able to change the last bits: 5 of 5 attempts detected it, with 3 to 38 of 40 runs differing. Verified non-vacuous by reverting the BTreeMap and watching it fail. Closes #62 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011hcFjNDmHXZF8URGLku5zZ
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,157 @@
|
||||
//! Determinism across *processes*, which an in-process test cannot see.
|
||||
//!
|
||||
//! Rust seeds its default hasher once per process, so every `HashMap`
|
||||
//! iteration order is fixed for a run and varies between runs. A test that
|
||||
//! compares results within one process therefore cannot detect a float sum
|
||||
//! whose order comes from a map — all its samples share one seed.
|
||||
//!
|
||||
//! That is not hypothetical. `tests/determinism.rs` compares four thread counts
|
||||
//! inside one process and passed throughout, while `posterior_of` was returning
|
||||
//! two distinct bit patterns across 40 separate runs on identical input.
|
||||
//!
|
||||
//! This re-executes the test binary and compares `f64::to_bits`.
|
||||
|
||||
use std::{env, process::Command};
|
||||
|
||||
use smallvec::smallvec;
|
||||
use trueskill_tt::{
|
||||
ConstantDrift, ConvergenceOptions, Event, History, Member, NullObserver, Outcome, Team,
|
||||
UnknownKeys,
|
||||
};
|
||||
|
||||
/// Set in the child so it reports instead of re-spawning.
|
||||
const CHILD: &str = "TSTT_DETERMINISM_CHILD";
|
||||
|
||||
const RUNS: usize = 40;
|
||||
|
||||
type H = History<i64, ConstantDrift, NullObserver, String>;
|
||||
|
||||
fn fitted() -> H {
|
||||
let mut h: H = History::builder_with_key()
|
||||
.mu(0.0)
|
||||
.sigma(6.0)
|
||||
.beta(1.0)
|
||||
.score_sigma(2.0)
|
||||
.drift(ConstantDrift(0.05))
|
||||
.unknown_keys(UnknownKeys::Prior)
|
||||
.convergence(ConvergenceOptions {
|
||||
max_iter: 20_000,
|
||||
epsilon: 1e-13,
|
||||
alpha: 1.0,
|
||||
})
|
||||
.build();
|
||||
|
||||
let mut events = Vec::new();
|
||||
for t in 0..12i64 {
|
||||
for k in 0..6usize {
|
||||
let a = format!("p{}", (t as usize * 6 + k) % 10);
|
||||
let b = format!("p{}", (t as usize * 6 + k + 4) % 10);
|
||||
events.push(Event {
|
||||
time: t,
|
||||
teams: smallvec![
|
||||
Team::with_members([Member::new(a)]),
|
||||
Team::with_members([Member::new(b)]),
|
||||
],
|
||||
outcome: Outcome::scores([3.0, 1.0]),
|
||||
});
|
||||
}
|
||||
}
|
||||
h.add_events(events).unwrap();
|
||||
assert!(h.converge().unwrap().converged);
|
||||
h
|
||||
}
|
||||
|
||||
/// Every quantity that could plausibly depend on iteration order, as bits.
|
||||
fn fingerprint() -> String {
|
||||
let h = fitted();
|
||||
|
||||
// Unknown keys with UNEQUAL but COMPARABLE coefficients, which is what
|
||||
// makes the sum order-sensitive.
|
||||
//
|
||||
// Equal terms sum order-independently and would make this pass vacuously.
|
||||
// Terms of wildly different magnitudes are no better: the small ones fall
|
||||
// below the running total's ULP and are absorbed whatever the order —
|
||||
// measured, spreading these over nine decades dropped the detection rate
|
||||
// to roughly one run in forty. Comparable sizes keep every term able to
|
||||
// change the last bits.
|
||||
let ghosts: Vec<String> = (0..24).map(|i| format!("ghost{i}")).collect();
|
||||
let mut terms: Vec<(&String, f64)> = ghosts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, k)| (k, 1.0 + i as f64 * 0.37))
|
||||
.collect();
|
||||
let known = "p0".to_string();
|
||||
terms.push((&known, -1.0));
|
||||
|
||||
let posterior = h.posterior_of(&terms).unwrap();
|
||||
|
||||
let a = "p0".to_string();
|
||||
let b = "p1".to_string();
|
||||
let target = [(&a, 1.0), (&b, -1.0)];
|
||||
let teams: [&[&String]; 2] = [&[&a], &[&b]];
|
||||
let evr = h.expected_variance_reduction(&teams, &target).unwrap();
|
||||
|
||||
let curves = h.learning_curves();
|
||||
let mut curve_bits: u64 = 0;
|
||||
let mut keys: Vec<&String> = curves.keys().collect();
|
||||
keys.sort();
|
||||
for key in keys {
|
||||
for (t, g) in &curves[key] {
|
||||
curve_bits ^= (*t as u64).rotate_left(17)
|
||||
^ g.mu().to_bits().rotate_left(31)
|
||||
^ g.sigma().to_bits();
|
||||
}
|
||||
}
|
||||
|
||||
format!(
|
||||
"post={:016x} evr={:016x} le={:016x} curves={curve_bits:016x}",
|
||||
posterior.sigma().to_bits(),
|
||||
evr.to_bits(),
|
||||
h.log_evidence().to_bits(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn results_are_identical_across_processes() {
|
||||
if env::var(CHILD).is_ok() {
|
||||
println!("FINGERPRINT {}", fingerprint());
|
||||
return;
|
||||
}
|
||||
|
||||
let exe = env::current_exe().expect("current exe");
|
||||
let mut seen: Vec<String> = Vec::new();
|
||||
|
||||
for run in 0..RUNS {
|
||||
let out = Command::new(&exe)
|
||||
.args([
|
||||
"results_are_identical_across_processes",
|
||||
"--exact",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, "1")
|
||||
.output()
|
||||
.expect("spawn child");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"child {run} failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
let line = stdout
|
||||
.lines()
|
||||
.find_map(|l| l.strip_prefix("FINGERPRINT "))
|
||||
.unwrap_or_else(|| panic!("child {run} printed no fingerprint:\n{stdout}"))
|
||||
.to_string();
|
||||
seen.push(line);
|
||||
}
|
||||
|
||||
let first = &seen[0];
|
||||
let differing: Vec<&String> = seen.iter().filter(|s| *s != first).collect();
|
||||
assert!(
|
||||
differing.is_empty(),
|
||||
"results differ across processes on identical input.\n {} of {RUNS} runs differed\n \
|
||||
first: {first}\n differing: {}",
|
||||
differing.len(),
|
||||
differing[0]
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user