diff --git a/tests/additive_model.rs b/tests/additive_model.rs new file mode 100644 index 0000000..9738d3c --- /dev/null +++ b/tests/additive_model.rs @@ -0,0 +1,142 @@ +//! What an additive model does to uncertainty, and why "add the marginals" is +//! unsafe in one direction and merely wasteful in the other. +//! +//! Structurally this is the shape a joint player/layout model takes: every +//! observation measures a *sum* of nodes against a reference, so the data pins +//! differences and leaves the overall level to the prior. That is the classic +//! rating-scale indeterminacy, not a defect. +//! +//! The consequence for a consumer is that combining marginals is wrong in +//! opposite directions depending on the combination, which is worth pinning +//! because the unsafe direction is not the one you would guess: +//! +//! - **Differences** (`a - b`): the shared level cancels, so the exact width is +//! small — and adding marginals lands within a couple of percent of it here, +//! because the loopy underestimate offsets the ignored correlation. +//! - **Sums** (`a + b`): the shared level does *not* cancel, so the exact width +//! is large, and adding marginals is roughly five times too narrow. That is +//! overconfident, and it is the direction that publishes a claim the data +//! does not support. + +use smallvec::smallvec; +use trueskill_tt::{ConstantDrift, ConvergenceOptions, Event, History, Member, Outcome, Team}; + +#[test] +fn additive_structure_makes_sums_wide_and_differences_tight() { + // Structurally like ustat: every round is (player + hole) measured against + // a fixed reference. Only SUMS are pinned by the data; the split between + // player and hole is pinned only by the prior. + let players = ["p0", "p1", "p2"]; + let holes = ["h0", "h1"]; + + let mut h: History = History::builder() + .mu(0.0) + .sigma(6.0) + .beta(1.0) + .score_sigma(2.0) + .drift(ConstantDrift(0.0)) + .convergence(ConvergenceOptions { + max_iter: 20_000, + epsilon: 1e-12, + alpha: 1.0, + }) + .build(); + + let mut seed = 3u64; + let mut rnd = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + // true skills, so we know what the data encodes + let truth_p = [2.0, 0.0, -2.0]; + let truth_h = [1.0, -1.0]; + + let mut events = Vec::new(); + for _ in 0..60 { + let p = (rnd() as usize) % 3; + let q = (rnd() as usize) % 2; + let noise = ((rnd() % 1000) as f64 / 1000.0 - 0.5) * 2.0; + let score = truth_p[p] + truth_h[q] + noise; + events.push(Event { + time: 1, + teams: smallvec![ + Team::with_members([Member::new(players[p]), Member::new(holes[q])]), + Team::with_members([Member::new("reference")]), + ], + outcome: Outcome::scores([score, 0.0]), + }); + } + h.add_events(events).unwrap(); + let r = h.converge().unwrap(); + assert!(r.converged, "{:?}", r.final_step); + + println!("\n== marginals (what current_skill reports) =="); + for k in players.iter().chain(holes.iter()) { + let g = h.current_skill(k).unwrap(); + println!(" {k}: mu {:>8.4} sigma {:>8.4}", g.mu(), g.sigma()); + } + + println!("\n== the same nodes via posterior_of (exact marginal) =="); + for k in players.iter().chain(holes.iter()) { + let g = h.posterior_of(&[(k, 1.0)]).unwrap(); + println!(" {k}: mu {:>8.4} sigma {:>8.4}", g.mu(), g.sigma()); + } + + println!("\n== combinations the data actually pins =="); + for (label, terms) in [ + ("p0 + h0 (a round)", vec![(&"p0", 1.0), (&"h0", 1.0)]), + ( + "p0 - p1 (rank two players)", + vec![(&"p0", 1.0), (&"p1", -1.0)], + ), + ("p0 - p2", vec![(&"p0", 1.0), (&"p2", -1.0)]), + ( + "h0 - h1 (rank two holes)", + vec![(&"h0", 1.0), (&"h1", -1.0)], + ), + ] { + let joint = h.posterior_of(&terms).unwrap(); + // what a consumer gets today by adding marginals + let naive: f64 = terms + .iter() + .map(|(k, c)| c * c * h.current_skill(*k).unwrap().sigma().powi(2)) + .sum::() + .sqrt(); + println!( + " {label:<28} exact sigma {:>7.4} adding marginals {:>7.4} {:>5.2}x over", + joint.sigma(), + naive, + naive / joint.sigma() + ); + + let ratio = naive / joint.sigma(); + if label.contains('+') { + assert!( + ratio < 0.5, + "{label}: adding marginals should be badly OVERconfident for a \ + sum, got {ratio:.3}x" + ); + } else { + assert!( + (0.8..1.25).contains(&ratio), + "{label}: adding marginals happens to be close for a difference, \ + got {ratio:.3}x" + ); + } + } + + // A single node in an additive model is weakly identified: its exact + // posterior is far wider than message passing reports, because the level it + // shares with its partners is pinned only by the prior. + for k in players.iter().chain(holes.iter()) { + let bp = h.current_skill(k).unwrap().sigma(); + let exact = h.posterior_of(&[(k, 1.0)]).unwrap().sigma(); + assert!( + exact > 3.0 * bp, + "{k}: exact marginal {exact} should be much wider than the reported \ + {bp} in an additive model" + ); + } +}