feat: add filtered_log_evidence
Scores every event on what was known before it, rather than on priors that carry information from events which had not happened yet. This is the quantity HistoryBuilder::online promised and never delivered. The pass walks slices in time order carrying its own forward messages, and per slice runs the unmodified production sweep on a scratch copy whose backward message is left improper. Reusing iterate_to_convergence rather than reimplementing inference means a competitor playing twice at one time is handled by the same within-slice EP that converge() uses, instead of being approximated the way the old evidence paths approximated it. Nothing is stored on Skill and nothing on self is mutated, so the result is independent of whether converge() has run — the property a stored field cannot have.
This commit is contained in:
+40
-1
@@ -13,7 +13,7 @@ use crate::{
|
||||
sort_time,
|
||||
storage::CompetitorStore,
|
||||
time::Time,
|
||||
time_slice::{self, EventKind, TimeSlice},
|
||||
time_slice::{self, EventKind, FilteredStep, TimeSlice},
|
||||
tuple_gt, tuple_max,
|
||||
};
|
||||
|
||||
@@ -416,6 +416,45 @@ impl<T: Time, D: Drift<T>, O: Observer<T>, K: Eq + Hash + Clone> History<T, D, O
|
||||
self.log_evidence_internal(false, &targets)
|
||||
}
|
||||
|
||||
/// Walk the slices in time order carrying forward messages only.
|
||||
///
|
||||
/// This is the forward half of `iteration` with the backward half never
|
||||
/// run. It reads `self` and mutates nothing.
|
||||
fn filtered_pass(&self) -> Vec<(T, FilteredStep)> {
|
||||
let mut messages: HashMap<Index, Gaussian> = HashMap::new();
|
||||
|
||||
let mut pass = Vec::with_capacity(self.time_slices.len());
|
||||
|
||||
for slice in &self.time_slices {
|
||||
let step = slice.filtered_step(&messages, &self.agents);
|
||||
|
||||
for &(agent, posterior) in &step.posteriors {
|
||||
messages.insert(agent, posterior);
|
||||
}
|
||||
|
||||
pass.push((slice.time, step));
|
||||
}
|
||||
|
||||
pass
|
||||
}
|
||||
|
||||
/// Total log-evidence under forward-only (filtering) information.
|
||||
///
|
||||
/// Each event is scored using only what was known before it, which is the
|
||||
/// right quantity for prequential scoring and model comparison. Contrast
|
||||
/// `log_evidence`, whose per-event priors carry information from events
|
||||
/// that had not happened yet.
|
||||
///
|
||||
/// Runs a full forward pass per call and caches nothing. The result does
|
||||
/// not depend on whether `converge` has been called.
|
||||
#[must_use]
|
||||
pub fn filtered_log_evidence(&self) -> f64 {
|
||||
self.filtered_pass()
|
||||
.iter()
|
||||
.map(|(_, step)| step.log_evidence)
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Draw-probability quality metric for the given teams (key slices).
|
||||
///
|
||||
/// Values range roughly [0, 1]; 1 == perfectly matched. Supports any
|
||||
|
||||
+81
-6
@@ -48,7 +48,7 @@ pub enum EventKind {
|
||||
Scored { score_sigma: f64 },
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
struct Item {
|
||||
agent: Index,
|
||||
likelihood: Gaussian,
|
||||
@@ -72,13 +72,13 @@ impl Item {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
struct Team {
|
||||
items: Vec<Item>,
|
||||
output: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct Event {
|
||||
teams: Vec<Team>,
|
||||
log_evidence: f64,
|
||||
@@ -189,6 +189,17 @@ struct EventUpdate {
|
||||
likelihoods: Vec<Vec<Gaussian>>,
|
||||
}
|
||||
|
||||
/// One slice's worth of forward-only inference.
|
||||
///
|
||||
/// `posteriors` doubles as the outgoing forward message: the scratch sweep
|
||||
/// never writes `backward`, so it stays `N_INF`, and `Skill::posterior()`
|
||||
/// and `forward_prior_out` are then the same product.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct FilteredStep {
|
||||
pub(crate) log_evidence: f64,
|
||||
pub(crate) posteriors: Vec<(Index, Gaussian)>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TimeSlice<T: Time = i64> {
|
||||
pub(crate) events: Vec<Event>,
|
||||
@@ -500,13 +511,13 @@ impl<T: Time> TimeSlice<T> {
|
||||
/// Iterate this slice alone until its posteriors stop moving, returning
|
||||
/// the number of iterations taken.
|
||||
///
|
||||
/// Only used by tests: production convergence is driven across slices by
|
||||
/// `History::converge`.
|
||||
/// Used by `filtered_step` to drive a scratch copy of the slice, and by
|
||||
/// tests. Production convergence across slices is driven by
|
||||
/// `History::converge`, which calls `iteration` directly.
|
||||
///
|
||||
/// Honours `self.convergence`; it previously hard-coded an epsilon and a
|
||||
/// 20-iteration cap that matched neither `ConvergenceOptions` nor the
|
||||
/// schedule default.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn iterate_to_convergence<D: Drift<T>>(
|
||||
&mut self,
|
||||
agents: &CompetitorStore<T, D>,
|
||||
@@ -574,6 +585,70 @@ impl<T: Time> TimeSlice<T> {
|
||||
self.iteration(0, agents);
|
||||
}
|
||||
|
||||
/// Run this slice's events on forward (filtering) information alone.
|
||||
///
|
||||
/// `incoming` holds each competitor's forward message out of their
|
||||
/// previous appearance; a competitor absent from it starts at their
|
||||
/// configured prior. The sweep runs on a scratch copy, so the real slice
|
||||
/// is untouched — which is what makes the filtered estimates independent
|
||||
/// of whether `History::converge` has run.
|
||||
pub(crate) fn filtered_step<D: Drift<T>>(
|
||||
&self,
|
||||
incoming: &HashMap<Index, Gaussian>,
|
||||
agents: &CompetitorStore<T, D>,
|
||||
) -> FilteredStep {
|
||||
let mut scratch = TimeSlice {
|
||||
events: self.events.clone(),
|
||||
skills: SkillStore::new(),
|
||||
time: self.time,
|
||||
p_draw: self.p_draw,
|
||||
convergence: self.convergence,
|
||||
arena: ScratchArena::new(),
|
||||
color_groups: ColorGroups::new(),
|
||||
color_groups_dirty: true,
|
||||
};
|
||||
|
||||
for event in &mut scratch.events {
|
||||
for team in &mut event.teams {
|
||||
for item in &mut team.items {
|
||||
item.likelihood = N_INF;
|
||||
}
|
||||
}
|
||||
|
||||
event.log_evidence = 0.0;
|
||||
}
|
||||
|
||||
for (agent, skill) in self.skills.iter() {
|
||||
let rating = &agents[agent].rating;
|
||||
|
||||
let forward = match incoming.get(&agent) {
|
||||
Some(message) => message.forget(rating.drift.variance_for_elapsed(skill.elapsed)),
|
||||
None => rating.prior,
|
||||
};
|
||||
|
||||
scratch.skills.insert(
|
||||
agent,
|
||||
Skill {
|
||||
forward,
|
||||
backward: N_INF,
|
||||
likelihood: N_INF,
|
||||
elapsed: skill.elapsed,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
scratch.iterate_to_convergence(agents);
|
||||
|
||||
FilteredStep {
|
||||
log_evidence: scratch.events.iter().map(|event| event.log_evidence).sum(),
|
||||
posteriors: scratch
|
||||
.skills
|
||||
.iter()
|
||||
.map(|(agent, skill)| (agent, skill.posterior()))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn log_evidence<D: Drift<T>>(
|
||||
&self,
|
||||
targets: &[Index],
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
//! Forward-only (filtering) estimates: what the model knew at the time,
|
||||
//! as opposed to the smoothed posteriors `learning_curve` reports.
|
||||
|
||||
use smallvec::smallvec;
|
||||
use trueskill_tt::{Event, History, Member, Outcome, Team};
|
||||
|
||||
/// `games` one-on-one matches at successive times, won by "a" every time.
|
||||
///
|
||||
/// This is the fixture from issue #19, where `online(true)` reported
|
||||
/// `games * ln(0.5)`.
|
||||
fn repeated_winner(games: i64) -> History {
|
||||
let mut history = History::builder().build();
|
||||
|
||||
for time in 1..=games {
|
||||
history
|
||||
.add_events([Event {
|
||||
time,
|
||||
teams: smallvec![
|
||||
Team::with_members([Member::new("a")]),
|
||||
Team::with_members([Member::new("b")]),
|
||||
],
|
||||
outcome: Outcome::winner(0, 2),
|
||||
}])
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
history
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filtered_evidence_sits_between_coin_flip_and_batch() {
|
||||
let mut history = repeated_winner(5);
|
||||
|
||||
history.converge().unwrap();
|
||||
|
||||
let coin_flip = 5.0 * 0.5f64.ln();
|
||||
let batch = history.log_evidence();
|
||||
let filtered = history.filtered_log_evidence();
|
||||
|
||||
assert!(
|
||||
filtered > coin_flip,
|
||||
"filtered evidence {filtered} is at or below {coin_flip}, the all-coin-flip \
|
||||
value the inert online flag reported; game one is a coin flip but games two \
|
||||
through five are not"
|
||||
);
|
||||
|
||||
assert!(
|
||||
filtered < batch,
|
||||
"filtered evidence {filtered} is not below the smoothed {batch}; filtering \
|
||||
scores each game on strictly less information than smoothing does"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user