feat: add History::predict_margin for scored matchups
#48: every predict_* answers "who wins", and a consumer recording scores never asks that. It wants the interval on the result, and having none it hand-fitted a noise law whose fitted node weight came out at 0.0 — so the quoted sigma was 5.83 whether the competitor had forty rounds or none, against real residual spreads of 5.8 and 12.44. `predict_margin` composes the three things that make a scored result uncertain: the joint posterior over the competitors, their per-event performance noise, and the observation noise on the score. It widens as the model knows less — measured, sigma 2.48 against an opponent with forty rounds, 3.38 against one seen once, wider still against one never seen — which is the property the hand-fitted law lost. It is a margin, not a score, and that is not a shortcut. Scored ingestion reduces every event to `score_a - score_b` before inference, so the absolute level is discarded: shifting every score in a history by +100 or -1000 produces a bit-identical fit, verified. There is no information from which to predict what a competitor will *score*. Returning one would be a number derived entirely from the prior, which is exactly the plausible constant this crate keeps finding and removing. `posterior_of` now honours `UnknownKeys::Prior`, which gives #48 its second requirement — "I have never seen this competitor, here is the prior-informed answer". An unseen competitor shares no event with the slice, so it is independent by construction and its variance is additive rather than part of the solve. Worth recording for expectations: for a *margin* the joint buys little over adding marginals (2.4798 against 2.5112 here), because a margin is a difference and differences are where the loopy underestimate and the ignored correlation cancel. The gain here is having a predictive distribution at all. `posterior_of`'s correlation handling earns its keep on sums and single nodes instead — see tests/additive_model.rs. Closes #48 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011hcFjNDmHXZF8URGLku5zZ
This commit is contained in:
+94
-11
@@ -813,17 +813,19 @@ impl<T: Time, D: Drift<T>, O: Observer<T>, K: Eq + Hash + Clone> History<T, D, O
|
|||||||
|
|
||||||
let mut contrast = vec![0.0; order.len()];
|
let mut contrast = vec![0.0; order.len()];
|
||||||
let mut mean = 0.0;
|
let mut mean = 0.0;
|
||||||
|
// A competitor the slice has never seen shares no event with anything
|
||||||
|
// in it, so it is independent by construction and its contribution is
|
||||||
|
// simply additive rather than part of the solve.
|
||||||
|
let mut independent_variance = 0.0;
|
||||||
|
|
||||||
for (member, (key, coefficient)) in terms.iter().enumerate() {
|
for (member, (key, coefficient)) in terms.iter().enumerate() {
|
||||||
let index = self.keys.get(*key).ok_or(InferenceError::UnknownKey {
|
let row = self
|
||||||
team: 0,
|
.keys
|
||||||
member,
|
.get(*key)
|
||||||
key: format!("{key:?}"),
|
.and_then(|index| row_of.get(&index).map(|row| (index, *row)));
|
||||||
})?;
|
|
||||||
let row = *row_of.get(&index).ok_or(InferenceError::UnknownKey {
|
match row {
|
||||||
team: 0,
|
Some((index, row)) => {
|
||||||
member,
|
|
||||||
key: format!("{key:?}"),
|
|
||||||
})?;
|
|
||||||
contrast[row] += coefficient;
|
contrast[row] += coefficient;
|
||||||
mean += coefficient
|
mean += coefficient
|
||||||
* slice
|
* slice
|
||||||
@@ -833,17 +835,98 @@ impl<T: Time, D: Drift<T>, O: Observer<T>, K: Eq + Hash + Clone> History<T, D, O
|
|||||||
.posterior()
|
.posterior()
|
||||||
.mu();
|
.mu();
|
||||||
}
|
}
|
||||||
|
None => match self.unknown_keys {
|
||||||
|
crate::UnknownKeys::Prior => {
|
||||||
|
mean += coefficient * self.mu;
|
||||||
|
independent_variance += coefficient * coefficient * self.sigma * self.sigma;
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
return Err(InferenceError::UnknownKey {
|
||||||
|
team: 0,
|
||||||
|
member,
|
||||||
|
key: format!("{key:?}"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let z =
|
let z =
|
||||||
crate::joint::solve_spd(lambda, &contrast).ok_or(InferenceError::JointUnavailable {
|
crate::joint::solve_spd(lambda, &contrast).ok_or(InferenceError::JointUnavailable {
|
||||||
reason: "the precision matrix is not positive-definite, which means \
|
reason: "the precision matrix is not positive-definite, which means \
|
||||||
a competitor has neither a proper prior nor any evidence",
|
a competitor has neither a proper prior nor any evidence",
|
||||||
})?;
|
})?;
|
||||||
let variance: f64 = contrast.iter().zip(&z).map(|(c, z)| c * z).sum();
|
let variance: f64 =
|
||||||
|
contrast.iter().zip(&z).map(|(c, z)| c * z).sum::<f64>() + independent_variance;
|
||||||
|
|
||||||
Ok(Gaussian::from_mv(mean, variance))
|
Ok(Gaussian::from_mv(mean, variance))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Predictive distribution of the score margin between two teams.
|
||||||
|
///
|
||||||
|
/// Answers "what will the gap be, and how wide is that interval" for a
|
||||||
|
/// scored matchup, composing the three things that make it uncertain: how
|
||||||
|
/// unsure the model is about the competitors, their per-event performance
|
||||||
|
/// noise, and the observation noise on the score itself.
|
||||||
|
///
|
||||||
|
/// The interval widens as the model knows less. Measured on a fixture where
|
||||||
|
/// one opponent has forty rounds and another has one, the margin's sigma
|
||||||
|
/// goes from 2.48 to 3.38 — which is the property a caller most needs and
|
||||||
|
/// the one a hand-fitted noise law tends to lose.
|
||||||
|
///
|
||||||
|
/// # Why a margin rather than a score
|
||||||
|
///
|
||||||
|
/// The model never sees an absolute score. Scored ingestion reduces each
|
||||||
|
/// event to `score_a - score_b` before inference, so shifting every score
|
||||||
|
/// in a history by a constant produces a bit-identical fit. There is
|
||||||
|
/// therefore no information from which to predict what a competitor will
|
||||||
|
/// *score*; only what the gap between two of them will be. Asking for an
|
||||||
|
/// absolute score would return a number derived entirely from the prior,
|
||||||
|
/// which is the kind of plausible constant this crate tries not to hand out.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// `MismatchedShape` unless exactly two teams are supplied, `EmptyTeam`,
|
||||||
|
/// `UnknownKey` under [`UnknownKeys::Reject`](crate::UnknownKeys::Reject),
|
||||||
|
/// and `JointUnavailable` if the latest slice holds ranked events.
|
||||||
|
pub fn predict_margin(&self, teams: &[&[&K]]) -> Result<Gaussian, InferenceError>
|
||||||
|
where
|
||||||
|
K: std::fmt::Debug,
|
||||||
|
{
|
||||||
|
if teams.len() != 2 {
|
||||||
|
return Err(InferenceError::MismatchedShape {
|
||||||
|
kind: "predict_margin takes exactly 2 teams",
|
||||||
|
expected: 2,
|
||||||
|
got: teams.len(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut terms: Vec<(&K, f64)> = Vec::new();
|
||||||
|
let mut performance_noise = 0.0;
|
||||||
|
|
||||||
|
for (team_idx, team) in teams.iter().enumerate() {
|
||||||
|
if team.is_empty() {
|
||||||
|
return Err(InferenceError::EmptyTeam { team: team_idx });
|
||||||
|
}
|
||||||
|
let sign = if team_idx == 0 { 1.0 } else { -1.0 };
|
||||||
|
for key in team.iter() {
|
||||||
|
terms.push((*key, sign));
|
||||||
|
// Each member contributes its own performance noise to the
|
||||||
|
// margin regardless of which side it is on.
|
||||||
|
let beta = self
|
||||||
|
.keys
|
||||||
|
.get(*key)
|
||||||
|
.map_or(self.beta, |index| self.agents[index].rating.beta);
|
||||||
|
performance_noise += beta * beta;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let skill_gap = self.posterior_of(&terms)?;
|
||||||
|
let variance = skill_gap.sigma().powi(2) + performance_noise + self.score_sigma.powi(2);
|
||||||
|
|
||||||
|
Ok(Gaussian::from_mv(skill_gap.mu(), variance))
|
||||||
|
}
|
||||||
|
|
||||||
/// Expected information gain of running this matchup, in nats.
|
/// Expected information gain of running this matchup, in nats.
|
||||||
///
|
///
|
||||||
/// Answers "which comparison should I run next" rather than "who will
|
/// Answers "which comparison should I run next" rather than "who will
|
||||||
|
|||||||
@@ -0,0 +1,153 @@
|
|||||||
|
//! `predict_margin`: the predictive distribution of a scored matchup.
|
||||||
|
|
||||||
|
use smallvec::smallvec;
|
||||||
|
use trueskill_tt::{
|
||||||
|
ConstantDrift, ConvergenceOptions, Event, History, InferenceError, Member, Outcome, Team,
|
||||||
|
UnknownKeys,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn builder(
|
||||||
|
policy: UnknownKeys,
|
||||||
|
) -> History<i64, ConstantDrift, trueskill_tt::NullObserver, &'static str> {
|
||||||
|
History::builder()
|
||||||
|
.mu(0.0)
|
||||||
|
.sigma(6.0)
|
||||||
|
.beta(1.0)
|
||||||
|
.score_sigma(2.0)
|
||||||
|
.drift(ConstantDrift(0.0))
|
||||||
|
.unknown_keys(policy)
|
||||||
|
.convergence(ConvergenceOptions {
|
||||||
|
max_iter: 5_000,
|
||||||
|
epsilon: 1e-12,
|
||||||
|
alpha: 1.0,
|
||||||
|
})
|
||||||
|
.build()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn round(a: &'static str, b: &'static str, sa: f64, sb: f64) -> Event<i64, &'static str> {
|
||||||
|
Event {
|
||||||
|
time: 1,
|
||||||
|
teams: smallvec![
|
||||||
|
Team::with_members([Member::new(a)]),
|
||||||
|
Team::with_members([Member::new(b)]),
|
||||||
|
],
|
||||||
|
outcome: Outcome::scores([sa, sb]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A history where "veteran" and "regular" are well observed and "novice"
|
||||||
|
/// appears once.
|
||||||
|
fn fitted(
|
||||||
|
policy: UnknownKeys,
|
||||||
|
) -> History<i64, ConstantDrift, trueskill_tt::NullObserver, &'static str> {
|
||||||
|
let mut h = builder(policy);
|
||||||
|
let mut events: Vec<_> = (0..40)
|
||||||
|
.map(|t| round("veteran", "regular", 10.0 + f64::from(t % 3), 5.0))
|
||||||
|
.collect();
|
||||||
|
events.push(round("veteran", "novice", 10.0, 6.0));
|
||||||
|
h.add_events(events).unwrap();
|
||||||
|
let _ = h.converge().unwrap();
|
||||||
|
h
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The property #48 exists for: the interval must widen when the model knows
|
||||||
|
/// less. Their hand-fitted noise law quoted the same sigma for a competitor
|
||||||
|
/// with forty rounds and one with none.
|
||||||
|
#[test]
|
||||||
|
fn the_interval_widens_as_the_model_knows_less() {
|
||||||
|
let h = fitted(UnknownKeys::Prior);
|
||||||
|
|
||||||
|
let well_known = h
|
||||||
|
.predict_margin(&[&[&"veteran"], &[&"regular"]])
|
||||||
|
.unwrap()
|
||||||
|
.sigma();
|
||||||
|
let thin = h
|
||||||
|
.predict_margin(&[&[&"veteran"], &[&"novice"]])
|
||||||
|
.unwrap()
|
||||||
|
.sigma();
|
||||||
|
let unseen = h
|
||||||
|
.predict_margin(&[&[&"veteran"], &[&"stranger"]])
|
||||||
|
.unwrap()
|
||||||
|
.sigma();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
well_known < thin && thin < unseen,
|
||||||
|
"margin width should grow as evidence thins: {well_known} < {thin} < {unseen}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// #48's second requirement: an unseen competitor is a legitimate question, not
|
||||||
|
/// an error, and the answer should come from the prior rather than be faked.
|
||||||
|
#[test]
|
||||||
|
fn an_unseen_competitor_is_answered_from_the_prior() {
|
||||||
|
let h = fitted(UnknownKeys::Prior);
|
||||||
|
let g = h.predict_margin(&[&[&"nobody"], &[&"no_one"]]).unwrap();
|
||||||
|
|
||||||
|
// Two unknowns: the gap is centred on zero and carries both priors plus
|
||||||
|
// both performance noises plus the observation noise.
|
||||||
|
assert!(g.mu().abs() < 1e-9, "mu {}", g.mu());
|
||||||
|
let expected = (2.0 * 36.0 + 2.0 * 1.0 + 4.0f64).sqrt();
|
||||||
|
assert!(
|
||||||
|
(g.sigma() - expected).abs() < 1e-9,
|
||||||
|
"sigma {} vs expected {expected}",
|
||||||
|
g.sigma()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reject_still_rejects() {
|
||||||
|
let h = fitted(UnknownKeys::Reject);
|
||||||
|
assert!(matches!(
|
||||||
|
h.predict_margin(&[&[&"veteran"], &[&"stranger"]]),
|
||||||
|
Err(InferenceError::UnknownKey { .. })
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The margin is the *difference*, so it must be antisymmetric in the teams.
|
||||||
|
#[test]
|
||||||
|
fn swapping_the_teams_negates_the_margin() {
|
||||||
|
let h = fitted(UnknownKeys::Prior);
|
||||||
|
let forward = h.predict_margin(&[&[&"veteran"], &[&"regular"]]).unwrap();
|
||||||
|
let reverse = h.predict_margin(&[&[&"regular"], &[&"veteran"]]).unwrap();
|
||||||
|
|
||||||
|
assert!((forward.mu() + reverse.mu()).abs() < 1e-9);
|
||||||
|
assert!((forward.sigma() - reverse.sigma()).abs() < 1e-12);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The predictive interval must be wider than the skill gap alone: it also
|
||||||
|
/// carries per-event performance noise and the observation noise.
|
||||||
|
#[test]
|
||||||
|
fn the_predictive_interval_exceeds_the_skill_uncertainty() {
|
||||||
|
let h = fitted(UnknownKeys::Prior);
|
||||||
|
let skill_gap = h
|
||||||
|
.posterior_of(&[(&"veteran", 1.0), (&"regular", -1.0)])
|
||||||
|
.unwrap();
|
||||||
|
let predictive = h.predict_margin(&[&[&"veteran"], &[&"regular"]]).unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
(predictive.mu() - skill_gap.mu()).abs() < 1e-12,
|
||||||
|
"means agree"
|
||||||
|
);
|
||||||
|
// beta^2 twice plus score_sigma^2 = 2 + 4.
|
||||||
|
let expected = (skill_gap.sigma().powi(2) + 6.0).sqrt();
|
||||||
|
assert!((predictive.sigma() - expected).abs() < 1e-12);
|
||||||
|
assert!(predictive.sigma() > skill_gap.sigma());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn shape_errors_are_reported() {
|
||||||
|
let h = fitted(UnknownKeys::Prior);
|
||||||
|
assert!(matches!(
|
||||||
|
h.predict_margin(&[&[&"veteran"]]),
|
||||||
|
Err(InferenceError::MismatchedShape {
|
||||||
|
expected: 2,
|
||||||
|
got: 1,
|
||||||
|
..
|
||||||
|
})
|
||||||
|
));
|
||||||
|
let empty: [&&str; 0] = [];
|
||||||
|
assert!(matches!(
|
||||||
|
h.predict_margin(&[&[&"veteran"], &empty]),
|
||||||
|
Err(InferenceError::EmptyTeam { team: 1 })
|
||||||
|
));
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user