trueskill-tt/tests/prediction_bounds.rs

//! Bounds that any correct implementation must satisfy, swept rather than
//! spot-checked.
//!
//! The crate's docs call the `ln k` ceiling "the sharpest available test of an
//! implementation", and record that an early prototype returned 4.77 nats. It
//! was violated again — 3.237828 nats against `ln 2` — because the existing
//! check sampled one fixture and the violation lives in a specific regime: a
//! large ratio between the widest and narrowest performance sigma, where the
//! shared prediction grid could not resolve the narrow density and returned
//! probabilities greater than one.
//!
//! A single fixture cannot defend a bound like this. A sweep can.

use trueskill_tt::{
    ConstantDrift, GameOptions, Gaussian, InferenceError, Rating, expected_information_gain,
};

type R = Rating<i64, ConstantDrift>;

/// How many random matchups the ceiling sweep draws.
///
/// Scaled by build profile rather than fixed. Each sample runs a full inference
/// pass per outcome, and that is about **19x** faster in release — measured,
/// 20 000 samples take 12.1s released against 23s for 2 000 in debug. `just
/// test` runs three debug feature combinations and one release one, so a fixed
/// count pays the slow price three times and the fast one once, which is
/// exactly backwards.
///
/// The debug run is here to prove the sweep still compiles and holds on a small
/// sample; the release run is the one that actually searches. The violation
/// this guards was found at a rate near 1.8%, so even the debug count expects
/// tens of hits in the regime.
#[cfg(debug_assertions)]
const SAMPLES: usize = 1_000;
#[cfg(not(debug_assertions))]
const SAMPLES: usize = 50_000;

/// Deterministic LCG, so a failure is reproducible from the printed seed.
struct Lcg(u64);

impl Lcg {
    fn next_f64(&mut self) -> f64 {
        self.0 = self
            .0
            .wrapping_mul(6_364_136_223_846_793_005)
            .wrapping_add(1_442_695_040_888_963_407);
        // Top 53 bits to [0, 1).
        ((self.0 >> 11) as f64) / ((1u64 << 53) as f64)
    }

    fn in_range(&mut self, lo: f64, hi: f64) -> f64 {
        lo + (hi - lo) * self.next_f64()
    }

    /// Log-uniform, so the sweep spends its samples across magnitudes rather
    /// than crowding the top of the range — the violations live at small sigma.
    fn log_uniform(&mut self, lo: f64, hi: f64) -> f64 {
        let t = self.next_f64();
        (lo.ln() + t * (hi.ln() - lo.ln())).exp()
    }
}

#[test]
fn information_gain_never_exceeds_the_entropy_of_the_outcome() {
    let mut rng = Lcg(0x5eed_1234_abcd_ef01);
    let ceiling = 2.0_f64.ln();
    let mut evaluated = 0usize;
    let mut refused = 0usize;

    for i in 0..SAMPLES {
        let mu_a = rng.in_range(-100.0, 100.0);
        let mu_b = rng.in_range(-100.0, 100.0);
        let sigma_a = rng.log_uniform(1e-4, 1e2);
        let sigma_b = rng.log_uniform(1e-4, 1e2);
        let beta = rng.log_uniform(1e-4, 1e1);

        let a = R::new(Gaussian::from_ms(mu_a, sigma_a), beta, ConstantDrift(0.0));
        let b = R::new(Gaussian::from_ms(mu_b, sigma_b), beta, ConstantDrift(0.0));
        let options = GameOptions {
            p_draw: 0.0,
            ..GameOptions::default()
        };

        match expected_information_gain(&[&[a], &[b]], &options) {
            Ok(gain) => {
                evaluated += 1;
                assert!(
                    gain.is_finite(),
                    "sample {i}: non-finite gain {gain} \
                     (mu {mu_a}, {mu_b}; sigma {sigma_a:e}, {sigma_b:e}; beta {beta:e})"
                );
                assert!(
                    gain >= 0.0,
                    "sample {i}: negative gain {gain} \
                     (mu {mu_a}, {mu_b}; sigma {sigma_a:e}, {sigma_b:e}; beta {beta:e})"
                );
                assert!(
                    gain <= ceiling + 1e-9,
                    "sample {i}: gain {gain} exceeds ln 2 = {ceiling} \
                     (mu {mu_a}, {mu_b}; sigma {sigma_a:e}, {sigma_b:e}; beta {beta:e})"
                );
            }
            // Refusing to answer is acceptable; answering wrongly is not.
            Err(InferenceError::GridTooCoarse { .. }) => refused += 1,
            Err(e) => panic!("sample {i}: unexpected error {e:?}"),
        }
    }

    // The sweep must actually exercise the function, not pass by refusing
    // everything.
    assert!(
        evaluated * 2 > SAMPLES,
        "only {evaluated} of {SAMPLES} samples were evaluated ({refused} refused); \
         the sweep is no longer testing anything"
    );
    // And it must still reach the regime where the ceiling was violated —
    // large sigma ratios, which is exactly where the grid now refuses. Without
    // this the sweep could drift into only-easy inputs and stop being a guard.
    assert!(
        refused > 0,
        "no sample reached the coarse-grid regime; the sweep no longer covers \
         the case that produced 3.24 nats"
    );
}

/// The regime that produced 3.237828 nats, pinned exactly.
#[test]
fn the_known_ceiling_violation_no_longer_answers_wrongly() {
    let a = R::new(
        Gaussian::from_ms(9.577_887_112_129_012, 0.000_132_507_526_585_134_38),
        0.000_307_235_559_013_096_2,
        ConstantDrift(0.0),
    );
    let b = R::new(
        Gaussian::from_ms(-14.114_932_828_525_696, 91.586_690_140_921_16),
        0.000_307_235_559_013_096_2,
        ConstantDrift(0.0),
    );
    let options = GameOptions {
        p_draw: 0.0,
        ..GameOptions::default()
    };

    match expected_information_gain(&[&[a], &[b]], &options) {
        Ok(gain) => assert!(
            gain <= 2.0_f64.ln() + 1e-9,
            "returned {gain}, over the ln 2 ceiling"
        ),
        Err(InferenceError::GridTooCoarse { needed, max }) => {
            assert!(needed > max, "needed {needed} should exceed max {max}");
        }
        Err(e) => panic!("unexpected error {e:?}"),
    }
}