diff --git a/src/history.rs b/src/history.rs index 752d4fc..3a7cb1b 100644 --- a/src/history.rs +++ b/src/history.rs @@ -13,7 +13,7 @@ use crate::{ sort_time, storage::CompetitorStore, time::Time, - time_slice::{self, EventKind, TimeSlice}, + time_slice::{self, EventKind, FilteredStep, TimeSlice}, tuple_gt, tuple_max, }; @@ -416,6 +416,45 @@ impl, O: Observer, K: Eq + Hash + Clone> History Vec<(T, FilteredStep)> { + let mut messages: HashMap = HashMap::new(); + + let mut pass = Vec::with_capacity(self.time_slices.len()); + + for slice in &self.time_slices { + let step = slice.filtered_step(&messages, &self.agents); + + for &(agent, posterior) in &step.posteriors { + messages.insert(agent, posterior); + } + + pass.push((slice.time, step)); + } + + pass + } + + /// Total log-evidence under forward-only (filtering) information. + /// + /// Each event is scored using only what was known before it, which is the + /// right quantity for prequential scoring and model comparison. Contrast + /// `log_evidence`, whose per-event priors carry information from events + /// that had not happened yet. + /// + /// Runs a full forward pass per call and caches nothing. The result does + /// not depend on whether `converge` has been called. + #[must_use] + pub fn filtered_log_evidence(&self) -> f64 { + self.filtered_pass() + .iter() + .map(|(_, step)| step.log_evidence) + .sum() + } + /// Draw-probability quality metric for the given teams (key slices). /// /// Values range roughly [0, 1]; 1 == perfectly matched. Supports any diff --git a/src/time_slice.rs b/src/time_slice.rs index f829d16..e9a9834 100644 --- a/src/time_slice.rs +++ b/src/time_slice.rs @@ -48,7 +48,7 @@ pub enum EventKind { Scored { score_sigma: f64 }, } -#[derive(Debug)] +#[derive(Clone, Debug)] struct Item { agent: Index, likelihood: Gaussian, @@ -72,13 +72,13 @@ impl Item { } } -#[derive(Debug)] +#[derive(Clone, Debug)] struct Team { items: Vec, output: f64, } -#[derive(Debug)] +#[derive(Clone, Debug)] pub(crate) struct Event { teams: Vec, log_evidence: f64, @@ -189,6 +189,17 @@ struct EventUpdate { likelihoods: Vec>, } +/// One slice's worth of forward-only inference. +/// +/// `posteriors` doubles as the outgoing forward message: the scratch sweep +/// never writes `backward`, so it stays `N_INF`, and `Skill::posterior()` +/// and `forward_prior_out` are then the same product. +#[derive(Debug)] +pub(crate) struct FilteredStep { + pub(crate) log_evidence: f64, + pub(crate) posteriors: Vec<(Index, Gaussian)>, +} + #[derive(Debug)] pub struct TimeSlice { pub(crate) events: Vec, @@ -500,13 +511,13 @@ impl TimeSlice { /// Iterate this slice alone until its posteriors stop moving, returning /// the number of iterations taken. /// - /// Only used by tests: production convergence is driven across slices by - /// `History::converge`. + /// Used by `filtered_step` to drive a scratch copy of the slice, and by + /// tests. Production convergence across slices is driven by + /// `History::converge`, which calls `iteration` directly. /// /// Honours `self.convergence`; it previously hard-coded an epsilon and a /// 20-iteration cap that matched neither `ConvergenceOptions` nor the /// schedule default. - #[cfg(test)] pub(crate) fn iterate_to_convergence>( &mut self, agents: &CompetitorStore, @@ -574,6 +585,70 @@ impl TimeSlice { self.iteration(0, agents); } + /// Run this slice's events on forward (filtering) information alone. + /// + /// `incoming` holds each competitor's forward message out of their + /// previous appearance; a competitor absent from it starts at their + /// configured prior. The sweep runs on a scratch copy, so the real slice + /// is untouched — which is what makes the filtered estimates independent + /// of whether `History::converge` has run. + pub(crate) fn filtered_step>( + &self, + incoming: &HashMap, + agents: &CompetitorStore, + ) -> FilteredStep { + let mut scratch = TimeSlice { + events: self.events.clone(), + skills: SkillStore::new(), + time: self.time, + p_draw: self.p_draw, + convergence: self.convergence, + arena: ScratchArena::new(), + color_groups: ColorGroups::new(), + color_groups_dirty: true, + }; + + for event in &mut scratch.events { + for team in &mut event.teams { + for item in &mut team.items { + item.likelihood = N_INF; + } + } + + event.log_evidence = 0.0; + } + + for (agent, skill) in self.skills.iter() { + let rating = &agents[agent].rating; + + let forward = match incoming.get(&agent) { + Some(message) => message.forget(rating.drift.variance_for_elapsed(skill.elapsed)), + None => rating.prior, + }; + + scratch.skills.insert( + agent, + Skill { + forward, + backward: N_INF, + likelihood: N_INF, + elapsed: skill.elapsed, + }, + ); + } + + scratch.iterate_to_convergence(agents); + + FilteredStep { + log_evidence: scratch.events.iter().map(|event| event.log_evidence).sum(), + posteriors: scratch + .skills + .iter() + .map(|(agent, skill)| (agent, skill.posterior())) + .collect(), + } + } + pub(crate) fn log_evidence>( &self, targets: &[Index], diff --git a/tests/filtered.rs b/tests/filtered.rs new file mode 100644 index 0000000..3642ac2 --- /dev/null +++ b/tests/filtered.rs @@ -0,0 +1,52 @@ +//! Forward-only (filtering) estimates: what the model knew at the time, +//! as opposed to the smoothed posteriors `learning_curve` reports. + +use smallvec::smallvec; +use trueskill_tt::{Event, History, Member, Outcome, Team}; + +/// `games` one-on-one matches at successive times, won by "a" every time. +/// +/// This is the fixture from issue #19, where `online(true)` reported +/// `games * ln(0.5)`. +fn repeated_winner(games: i64) -> History { + let mut history = History::builder().build(); + + for time in 1..=games { + history + .add_events([Event { + time, + teams: smallvec![ + Team::with_members([Member::new("a")]), + Team::with_members([Member::new("b")]), + ], + outcome: Outcome::winner(0, 2), + }]) + .unwrap(); + } + + history +} + +#[test] +fn filtered_evidence_sits_between_coin_flip_and_batch() { + let mut history = repeated_winner(5); + + history.converge().unwrap(); + + let coin_flip = 5.0 * 0.5f64.ln(); + let batch = history.log_evidence(); + let filtered = history.filtered_log_evidence(); + + assert!( + filtered > coin_flip, + "filtered evidence {filtered} is at or below {coin_flip}, the all-coin-flip \ + value the inert online flag reported; game one is a coin flip but games two \ + through five are not" + ); + + assert!( + filtered < batch, + "filtered evidence {filtered} is not below the smoothed {batch}; filtering \ + scores each game on strictly less information than smoothing does" + ); +}