trueskill-tt/tests/additive_model.rs

//! What an additive model does to uncertainty, and why "add the marginals" is
//! unsafe in one direction and merely wasteful in the other.
//!
//! Structurally this is the shape a joint player/layout model takes: every
//! observation measures a *sum* of nodes against a reference, so the data pins
//! differences and leaves the overall level to the prior. That is the classic
//! rating-scale indeterminacy, not a defect.
//!
//! The consequence for a consumer is that combining marginals is wrong in
//! opposite directions depending on the combination, which is worth pinning
//! because the unsafe direction is not the one you would guess:
//!
//! - **Differences** (`a - b`): the shared level cancels, so the exact width is
//!   small — and adding marginals lands within a couple of percent of it here,
//!   because the loopy underestimate offsets the ignored correlation.
//! - **Sums** (`a + b`): the shared level does *not* cancel, so the exact width
//!   is large, and adding marginals is roughly five times too narrow. That is
//!   overconfident, and it is the direction that publishes a claim the data
//!   does not support.

use smallvec::smallvec;
use trueskill_tt::{ConstantDrift, ConvergenceOptions, Event, History, Member, Outcome, Team};

#[test]
fn additive_structure_makes_sums_wide_and_differences_tight() {
    // Structurally like ustat: every round is (player + hole) measured against
    // a fixed reference. Only SUMS are pinned by the data; the split between
    // player and hole is pinned only by the prior.
    let players = ["p0", "p1", "p2"];
    let holes = ["h0", "h1"];

    let mut h: History = History::builder()
        .mu(0.0)
        .sigma(6.0)
        .beta(1.0)
        .score_sigma(2.0)
        .drift(ConstantDrift::new(0.0))
        .convergence(ConvergenceOptions {
            max_iter: 20_000,
            epsilon: 1e-12,
            alpha: 1.0,
        })
        .build();

    let mut seed = 3u64;
    let mut rnd = move || {
        seed ^= seed << 13;
        seed ^= seed >> 7;
        seed ^= seed << 17;
        seed
    };
    // true skills, so we know what the data encodes
    let truth_p = [2.0, 0.0, -2.0];
    let truth_h = [1.0, -1.0];

    let mut events = Vec::new();
    for _ in 0..60 {
        let p = (rnd() as usize) % 3;
        let q = (rnd() as usize) % 2;
        let noise = ((rnd() % 1000) as f64 / 1000.0 - 0.5) * 2.0;
        let score = truth_p[p] + truth_h[q] + noise;
        events.push(Event {
            time: 1,
            teams: smallvec![
                Team::with_members([Member::new(players[p]), Member::new(holes[q])]),
                Team::with_members([Member::new("reference")]),
            ],
            outcome: Outcome::scores([score, 0.0]),
        });
    }
    h.add_events(events).unwrap();
    let r = h.converge().unwrap();
    assert!(r.converged, "{:?}", r.final_step);

    println!("\n== marginals (what current_skill reports) ==");
    for k in players.iter().chain(holes.iter()) {
        let g = h.current_skill(k).unwrap();
        println!("  {k}: mu {:>8.4}  sigma {:>8.4}", g.mu(), g.sigma());
    }

    println!("\n== the same nodes via posterior_of (exact marginal) ==");
    for k in players.iter().chain(holes.iter()) {
        let g = h.joint().unwrap().posterior_of(&[(k, 1.0)]).unwrap();
        println!("  {k}: mu {:>8.4}  sigma {:>8.4}", g.mu(), g.sigma());
    }

    println!("\n== combinations the data actually pins ==");
    for (label, terms) in [
        ("p0 + h0 (a round)", vec![(&"p0", 1.0), (&"h0", 1.0)]),
        (
            "p0 - p1 (rank two players)",
            vec![(&"p0", 1.0), (&"p1", -1.0)],
        ),
        ("p0 - p2", vec![(&"p0", 1.0), (&"p2", -1.0)]),
        (
            "h0 - h1 (rank two holes)",
            vec![(&"h0", 1.0), (&"h1", -1.0)],
        ),
    ] {
        let joint = h.joint().unwrap().posterior_of(&terms).unwrap();
        // what a consumer gets today by adding marginals
        let naive: f64 = terms
            .iter()
            .map(|(k, c)| c * c * h.current_skill(*k).unwrap().sigma().powi(2))
            .sum::<f64>()
            .sqrt();
        println!(
            "  {label:<28} exact sigma {:>7.4}   adding marginals {:>7.4}   {:>5.2}x over",
            joint.sigma(),
            naive,
            naive / joint.sigma()
        );

        let ratio = naive / joint.sigma();
        if label.contains('+') {
            assert!(
                ratio < 0.5,
                "{label}: adding marginals should be badly OVERconfident for a \
                 sum, got {ratio:.3}x"
            );
        } else {
            assert!(
                (0.8..1.25).contains(&ratio),
                "{label}: adding marginals happens to be close for a difference, \
                 got {ratio:.3}x"
            );
        }
    }

    // A single node in an additive model is weakly identified: its exact
    // posterior is far wider than message passing reports, because the level it
    // shares with its partners is pinned only by the prior.
    for k in players.iter().chain(holes.iter()) {
        let bp = h.current_skill(k).unwrap().sigma();
        let exact = h
            .joint()
            .unwrap()
            .posterior_of(&[(k, 1.0)])
            .unwrap()
            .sigma();
        assert!(
            exact > 3.0 * bp,
            "{k}: exact marginal {exact} should be much wider than the reported \
             {bp} in an additive model"
        );
    }
}