trueskill-tt/tests/properties.rs

//! Property-based tests over generated histories.
//!
//! The golden suite pins exact values against the Python/Julia reference on a
//! handful of fixtures. These pin *invariants* over inputs nobody wrote by
//! hand, which is where the defects this crate has actually shipped were
//! hiding: a linear evidence product that underflowed only past ~1000 teams,
//! and a batching path no golden exercised because every golden ingests in one
//! call.

mod common;

use common::assert_finite;
use proptest::prelude::*;
use smallvec::smallvec;
use trueskill_tt::{ConvergenceOptions, Event, History, Member, Outcome, Team};

/// Distinct competitors, so no event pits someone against themselves.
fn pairs() -> impl Strategy<Value = Vec<(usize, usize)>> {
    prop::collection::vec((0usize..8, 0usize..8), 1..24)
        .prop_map(|v| v.into_iter().filter(|(a, b)| a != b).collect::<Vec<_>>())
        .prop_filter("needs at least one valid pair", |v| !v.is_empty())
}

const KEYS: [&str; 8] = ["a", "b", "c", "d", "e", "f", "g", "h"];

fn history_from(games: &[(usize, usize)]) -> History {
    let mut h = History::builder()
        .convergence(ConvergenceOptions {
            // 200 was not enough: the batched side stopped at the cap with a
            // step of 3.4e-9, so this test was comparing two truncated fits and
            // attributing the gap to ingestion order.
            max_iter: 20_000,
            epsilon: 1e-10,
            ..ConvergenceOptions::default()
        })
        .build();

    let events: Vec<Event<i64, &'static str>> = games
        .iter()
        .enumerate()
        .map(|(i, &(a, b))| Event {
            time: i as i64 + 1,
            teams: smallvec![
                Team::with_members([Member::new(KEYS[a])]),
                Team::with_members([Member::new(KEYS[b])]),
            ],
            outcome: Outcome::winner(0, 2),
        })
        .collect();

    h.add_events(events).unwrap();

    h
}

proptest! {
    #![proptest_config(ProptestConfig::with_cases(48))]

    /// Whatever the schedule of games, convergence must not produce NaN or an
    /// improper posterior. `converge` returns `NonFiniteResult` rather than
    /// silently reporting a NaN step as converged, so a break shows up here as
    /// either an Err or a non-finite curve point.
    #[test]
    fn converged_posteriors_are_always_finite(games in pairs()) {
        let mut h = history_from(&games);

        let _ = h.converge().unwrap();

        for key in KEYS {
            for (time, g) in h.learning_curve(key) {
                assert_finite(g, &format!("{key} at t={time}"));
            }
        }
    }

    /// Log-evidence is a log probability: finite, and never above zero.
    ///
    /// The linear-product implementation this replaced underflowed to zero on
    /// long chains, making `ln(0)` = -inf — finite-ness is the property that
    /// would have caught it.
    #[test]
    fn log_evidence_is_a_finite_log_probability(games in pairs()) {
        let mut h = history_from(&games);

        let _ = h.converge().unwrap();

        let batch = h.log_evidence();
        let filtered = h.filtered_log_evidence();

        prop_assert!(batch.is_finite(), "batch log-evidence {batch} is not finite");
        prop_assert!(batch <= 0.0, "batch log-evidence {batch} exceeds zero");
        prop_assert!(filtered.is_finite(), "filtered log-evidence {filtered} is not finite");
        prop_assert!(filtered <= 0.0, "filtered log-evidence {filtered} exceeds zero");
    }

    /// Filtered estimates must not depend on whether `converge` has run — the
    /// property the whole forward-only design rests on.
    #[test]
    fn filtered_evidence_is_invariant_to_convergence(games in pairs()) {
        let mut h = history_from(&games);

        let before = h.filtered_log_evidence();

        let _ = h.converge().unwrap();

        let after = h.filtered_log_evidence();

        prop_assert!(
            (before - after).abs() < 1e-8,
            "filtered evidence moved across converge(): {before} -> {after}"
        );
    }

    /// Ingesting the same games one at a time must reach the same fixed point
    /// as ingesting them in one call.
    #[test]
    fn ingestion_order_does_not_change_the_answer(games in pairs()) {
        let batched = {
            let mut h = history_from(&games);
            let report = h.converge().unwrap();
            prop_assert!(
                report.converged,
                "batched side stopped at {} iterations with step {:?}; comparing \
                 two fits that have not converged measures truncation, not order",
                report.iterations,
                report.final_step
            );
            h
        };

        let incremental = {
            let mut h = History::builder()
                .convergence(ConvergenceOptions {
                    max_iter: 20_000,
                    epsilon: 1e-10,
                    ..ConvergenceOptions::default()
                })
                .build();

            for (i, &(a, b)) in games.iter().enumerate() {
                h.add_events([Event {
                    time: i as i64 + 1,
                    teams: smallvec![
                        Team::with_members([Member::new(KEYS[a])]),
                        Team::with_members([Member::new(KEYS[b])]),
                    ],
                    outcome: Outcome::winner(0, 2),
                }])
                .unwrap();
            }

            let report = h.converge().unwrap();
            prop_assert!(
                report.converged,
                "incremental side stopped at {} iterations with step {:?}",
                report.iterations,
                report.final_step
            );
            h
        };

        for key in KEYS {
            let one = batched.current_skill(key);
            let other = incremental.current_skill(key);

            match (one, other) {
                (Some(one), Some(other)) => {
                    prop_assert!(
                        (one.mu() - other.mu()).abs() < 1e-6
                            && (one.sigma() - other.sigma()).abs() < 1e-6,
                        "{key}: batched mu={} sigma={}, incremental mu={} sigma={}",
                        one.mu(),
                        one.sigma(),
                        other.mu(),
                        other.sigma()
                    );
                }
                (None, None) => {}
                _ => prop_assert!(false, "{key} present in only one history"),
            }
        }
    }
}