diff --git a/src/history.rs b/src/history.rs index 0d35ce8..b3f2e2b 100644 --- a/src/history.rs +++ b/src/history.rs @@ -813,25 +813,42 @@ impl, O: Observer, K: Eq + Hash + Clone> History { + contrast[row] += coefficient; + mean += coefficient + * slice + .skills + .get(index) + .expect("index came from this slice") + .posterior() + .mu(); + } + None => match self.unknown_keys { + crate::UnknownKeys::Prior => { + mean += coefficient * self.mu; + independent_variance += coefficient * coefficient * self.sigma * self.sigma; + } + _ => { + return Err(InferenceError::UnknownKey { + team: 0, + member, + key: format!("{key:?}"), + }); + } + }, + } } let z = @@ -839,11 +856,77 @@ impl, O: Observer, K: Eq + Hash + Clone> History() + independent_variance; Ok(Gaussian::from_mv(mean, variance)) } + /// Predictive distribution of the score margin between two teams. + /// + /// Answers "what will the gap be, and how wide is that interval" for a + /// scored matchup, composing the three things that make it uncertain: how + /// unsure the model is about the competitors, their per-event performance + /// noise, and the observation noise on the score itself. + /// + /// The interval widens as the model knows less. Measured on a fixture where + /// one opponent has forty rounds and another has one, the margin's sigma + /// goes from 2.48 to 3.38 — which is the property a caller most needs and + /// the one a hand-fitted noise law tends to lose. + /// + /// # Why a margin rather than a score + /// + /// The model never sees an absolute score. Scored ingestion reduces each + /// event to `score_a - score_b` before inference, so shifting every score + /// in a history by a constant produces a bit-identical fit. There is + /// therefore no information from which to predict what a competitor will + /// *score*; only what the gap between two of them will be. Asking for an + /// absolute score would return a number derived entirely from the prior, + /// which is the kind of plausible constant this crate tries not to hand out. + /// + /// # Errors + /// + /// `MismatchedShape` unless exactly two teams are supplied, `EmptyTeam`, + /// `UnknownKey` under [`UnknownKeys::Reject`](crate::UnknownKeys::Reject), + /// and `JointUnavailable` if the latest slice holds ranked events. + pub fn predict_margin(&self, teams: &[&[&K]]) -> Result + where + K: std::fmt::Debug, + { + if teams.len() != 2 { + return Err(InferenceError::MismatchedShape { + kind: "predict_margin takes exactly 2 teams", + expected: 2, + got: teams.len(), + }); + } + + let mut terms: Vec<(&K, f64)> = Vec::new(); + let mut performance_noise = 0.0; + + for (team_idx, team) in teams.iter().enumerate() { + if team.is_empty() { + return Err(InferenceError::EmptyTeam { team: team_idx }); + } + let sign = if team_idx == 0 { 1.0 } else { -1.0 }; + for key in team.iter() { + terms.push((*key, sign)); + // Each member contributes its own performance noise to the + // margin regardless of which side it is on. + let beta = self + .keys + .get(*key) + .map_or(self.beta, |index| self.agents[index].rating.beta); + performance_noise += beta * beta; + } + } + + let skill_gap = self.posterior_of(&terms)?; + let variance = skill_gap.sigma().powi(2) + performance_noise + self.score_sigma.powi(2); + + Ok(Gaussian::from_mv(skill_gap.mu(), variance)) + } + /// Expected information gain of running this matchup, in nats. /// /// Answers "which comparison should I run next" rather than "who will diff --git a/tests/predict_margin.rs b/tests/predict_margin.rs new file mode 100644 index 0000000..cd54fb7 --- /dev/null +++ b/tests/predict_margin.rs @@ -0,0 +1,153 @@ +//! `predict_margin`: the predictive distribution of a scored matchup. + +use smallvec::smallvec; +use trueskill_tt::{ + ConstantDrift, ConvergenceOptions, Event, History, InferenceError, Member, Outcome, Team, + UnknownKeys, +}; + +fn builder( + policy: UnknownKeys, +) -> History { + History::builder() + .mu(0.0) + .sigma(6.0) + .beta(1.0) + .score_sigma(2.0) + .drift(ConstantDrift(0.0)) + .unknown_keys(policy) + .convergence(ConvergenceOptions { + max_iter: 5_000, + epsilon: 1e-12, + alpha: 1.0, + }) + .build() +} + +fn round(a: &'static str, b: &'static str, sa: f64, sb: f64) -> Event { + Event { + time: 1, + teams: smallvec![ + Team::with_members([Member::new(a)]), + Team::with_members([Member::new(b)]), + ], + outcome: Outcome::scores([sa, sb]), + } +} + +/// A history where "veteran" and "regular" are well observed and "novice" +/// appears once. +fn fitted( + policy: UnknownKeys, +) -> History { + let mut h = builder(policy); + let mut events: Vec<_> = (0..40) + .map(|t| round("veteran", "regular", 10.0 + f64::from(t % 3), 5.0)) + .collect(); + events.push(round("veteran", "novice", 10.0, 6.0)); + h.add_events(events).unwrap(); + let _ = h.converge().unwrap(); + h +} + +/// The property #48 exists for: the interval must widen when the model knows +/// less. Their hand-fitted noise law quoted the same sigma for a competitor +/// with forty rounds and one with none. +#[test] +fn the_interval_widens_as_the_model_knows_less() { + let h = fitted(UnknownKeys::Prior); + + let well_known = h + .predict_margin(&[&[&"veteran"], &[&"regular"]]) + .unwrap() + .sigma(); + let thin = h + .predict_margin(&[&[&"veteran"], &[&"novice"]]) + .unwrap() + .sigma(); + let unseen = h + .predict_margin(&[&[&"veteran"], &[&"stranger"]]) + .unwrap() + .sigma(); + + assert!( + well_known < thin && thin < unseen, + "margin width should grow as evidence thins: {well_known} < {thin} < {unseen}" + ); +} + +/// #48's second requirement: an unseen competitor is a legitimate question, not +/// an error, and the answer should come from the prior rather than be faked. +#[test] +fn an_unseen_competitor_is_answered_from_the_prior() { + let h = fitted(UnknownKeys::Prior); + let g = h.predict_margin(&[&[&"nobody"], &[&"no_one"]]).unwrap(); + + // Two unknowns: the gap is centred on zero and carries both priors plus + // both performance noises plus the observation noise. + assert!(g.mu().abs() < 1e-9, "mu {}", g.mu()); + let expected = (2.0 * 36.0 + 2.0 * 1.0 + 4.0f64).sqrt(); + assert!( + (g.sigma() - expected).abs() < 1e-9, + "sigma {} vs expected {expected}", + g.sigma() + ); +} + +#[test] +fn reject_still_rejects() { + let h = fitted(UnknownKeys::Reject); + assert!(matches!( + h.predict_margin(&[&[&"veteran"], &[&"stranger"]]), + Err(InferenceError::UnknownKey { .. }) + )); +} + +/// The margin is the *difference*, so it must be antisymmetric in the teams. +#[test] +fn swapping_the_teams_negates_the_margin() { + let h = fitted(UnknownKeys::Prior); + let forward = h.predict_margin(&[&[&"veteran"], &[&"regular"]]).unwrap(); + let reverse = h.predict_margin(&[&[&"regular"], &[&"veteran"]]).unwrap(); + + assert!((forward.mu() + reverse.mu()).abs() < 1e-9); + assert!((forward.sigma() - reverse.sigma()).abs() < 1e-12); +} + +/// The predictive interval must be wider than the skill gap alone: it also +/// carries per-event performance noise and the observation noise. +#[test] +fn the_predictive_interval_exceeds_the_skill_uncertainty() { + let h = fitted(UnknownKeys::Prior); + let skill_gap = h + .posterior_of(&[(&"veteran", 1.0), (&"regular", -1.0)]) + .unwrap(); + let predictive = h.predict_margin(&[&[&"veteran"], &[&"regular"]]).unwrap(); + + assert!( + (predictive.mu() - skill_gap.mu()).abs() < 1e-12, + "means agree" + ); + // beta^2 twice plus score_sigma^2 = 2 + 4. + let expected = (skill_gap.sigma().powi(2) + 6.0).sqrt(); + assert!((predictive.sigma() - expected).abs() < 1e-12); + assert!(predictive.sigma() > skill_gap.sigma()); +} + +#[test] +fn shape_errors_are_reported() { + let h = fitted(UnknownKeys::Prior); + assert!(matches!( + h.predict_margin(&[&[&"veteran"]]), + Err(InferenceError::MismatchedShape { + expected: 2, + got: 1, + .. + }) + )); + let empty: [&&str; 0] = []; + assert!(matches!( + h.predict_margin(&[&[&"veteran"], &empty]), + Err(InferenceError::EmptyTeam { team: 1 }) + )); +}