feat: complete the evidence matrix and add current_skills
Three of four corners of the evidence matrix existed. The missing one was forward-only *and* key-restricted — which is exactly what per-competitor prequential scoring needs, the intersection of the two workloads `log_evidence_for` and `filtered_log_evidence` are each documented for. `filtered_log_evidence_for` fills it. It is not `log_evidence_internal(true, targets)`: that path selects `skill.forward` as the prior, which stops being a filtering quantity once `iteration` has run a backward sweep. It goes through `filtered_pass` like its unrestricted sibling, with the restriction applied to which events are *scored*, never to which are *run* — so it is a held-out score under the real history, not a score under a counterfactual one where nobody else played. Key resolution for both `*_for` accessors now shares `resolve_targets`, so they cannot drift apart on how an unknown key is reported. `current_skills` is the plural of `current_skill`. Building a leaderboard previously meant materialising every competitor's full smoothed curve via `learning_curves` and reading the last point of each. Tests carry controls in both directions: naming every competitor must recover the unrestricted value (catching a filter that drops too much), and the restricted forward-only value must differ from the restricted smoothed one (catching an alias). Refs #70. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011hcFjNDmHXZF8URGLku5zZ
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
//! The evidence accessors span two independent axes — smoothed vs forward-only,
|
||||
//! all-keys vs key-restricted — and all four corners must exist and differ.
|
||||
//!
|
||||
//! `filtered_log_evidence_for` was the missing corner: the one a per-competitor
|
||||
//! prequential score needs.
|
||||
|
||||
use trueskill_tt::{
|
||||
ConstantDrift, Event, History, InferenceError, Member, NullObserver, Outcome, Team,
|
||||
};
|
||||
|
||||
type H = History<i64, ConstantDrift, NullObserver, &'static str>;
|
||||
|
||||
/// Two disjoint cohorts, so a key restriction is guaranteed to leave events out.
|
||||
fn two_cohorts() -> H {
|
||||
let mut h = H::default();
|
||||
let mut events = Vec::new();
|
||||
for t in 1..=6 {
|
||||
for (x, y) in [("a", "b"), ("c", "d")] {
|
||||
events.push(Event {
|
||||
time: t,
|
||||
teams: [
|
||||
Team::with_members([Member::new(x)]),
|
||||
Team::with_members([Member::new(y)]),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
outcome: Outcome::winner(0, 2),
|
||||
});
|
||||
}
|
||||
}
|
||||
h.add_events(events).expect("fixture ingests");
|
||||
h.converge().expect("fixture converges");
|
||||
h
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_four_corners_are_distinct_quantities() {
|
||||
let h = two_cohorts();
|
||||
|
||||
let smoothed_all = h.log_evidence();
|
||||
let smoothed_ab = h.log_evidence_for(&[&"a", &"b"]).unwrap();
|
||||
let filtered_all = h.filtered_log_evidence();
|
||||
let filtered_ab = h.filtered_log_evidence_for(&[&"a", &"b"]).unwrap();
|
||||
|
||||
for (name, v) in [
|
||||
("smoothed_all", smoothed_all),
|
||||
("smoothed_ab", smoothed_ab),
|
||||
("filtered_all", filtered_all),
|
||||
("filtered_ab", filtered_ab),
|
||||
] {
|
||||
assert!(
|
||||
v.is_finite() && v <= 0.0,
|
||||
"{name} = {v} is not a log probability"
|
||||
);
|
||||
}
|
||||
|
||||
// Restricting to one cohort must drop the other cohort's events. Half the
|
||||
// events, and the two cohorts are symmetric, so it lands near half.
|
||||
assert!(
|
||||
smoothed_ab > smoothed_all,
|
||||
"restricting must drop evidence terms: {smoothed_ab} vs {smoothed_all}"
|
||||
);
|
||||
assert!(filtered_ab > filtered_all);
|
||||
|
||||
// The forward-only corner is a genuinely different quantity from the
|
||||
// smoothed one, not an alias for it.
|
||||
assert!(
|
||||
(filtered_ab - smoothed_ab).abs() > 1e-9,
|
||||
"filtered and smoothed restricted evidence coincide ({filtered_ab} vs {smoothed_ab}); \
|
||||
one of them is not computing what it claims"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restricting_to_both_cohorts_recovers_the_unrestricted_value() {
|
||||
let h = two_cohorts();
|
||||
|
||||
// Control on the filter itself: naming every competitor must restrict
|
||||
// nothing, so this catches a filter that drops events it should keep.
|
||||
let all_named = h
|
||||
.filtered_log_evidence_for(&[&"a", &"b", &"c", &"d"])
|
||||
.unwrap();
|
||||
assert!(
|
||||
(all_named - h.filtered_log_evidence()).abs() < 1e-12,
|
||||
"naming everyone changed the answer: {all_named} vs {}",
|
||||
h.filtered_log_evidence()
|
||||
);
|
||||
}
|
||||
|
||||
/// The restriction selects *events*, not competitors: naming one member of a
|
||||
/// pair that only ever plays each other selects the same events as naming both.
|
||||
#[test]
|
||||
fn naming_either_member_of_a_pair_selects_the_same_events() {
|
||||
let h = two_cohorts();
|
||||
|
||||
let ab = h.filtered_log_evidence_for(&[&"a"]).unwrap();
|
||||
let ab_pair = h.filtered_log_evidence_for(&[&"a", &"b"]).unwrap();
|
||||
assert!(
|
||||
(ab - ab_pair).abs() < 1e-12,
|
||||
"a and b only ever play each other, so naming either or both selects \
|
||||
the same events: {ab} vs {ab_pair}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_key_is_an_error_here_too() {
|
||||
let h = two_cohorts();
|
||||
|
||||
let err = h
|
||||
.filtered_log_evidence_for(&[&"typo"])
|
||||
.expect_err("unknown key");
|
||||
assert!(matches!(err, InferenceError::UnknownKey { .. }), "{err:?}");
|
||||
|
||||
// Control: the same call on a known key succeeds.
|
||||
h.filtered_log_evidence_for(&[&"a"]).expect("a is known");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_skills_agrees_with_current_skill() {
|
||||
let h = two_cohorts();
|
||||
|
||||
let all = h.current_skills();
|
||||
assert_eq!(all.len(), 4, "four competitors played");
|
||||
|
||||
for key in ["a", "b", "c", "d"] {
|
||||
let one = h.current_skill(key).expect("played");
|
||||
let from_map = all[key];
|
||||
assert_eq!(
|
||||
(one.mu(), one.sigma()),
|
||||
(from_map.mu(), from_map.sigma()),
|
||||
"current_skills disagrees with current_skill for {key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_skills_omits_a_registered_but_unplayed_competitor() {
|
||||
let mut h = two_cohorts();
|
||||
h.register(Member::new("e")).expect("e is new");
|
||||
|
||||
let all = h.current_skills();
|
||||
assert!(
|
||||
!all.contains_key("e"),
|
||||
"a competitor with no appearances has no posterior to report"
|
||||
);
|
||||
assert!(
|
||||
h.current_skill("e").is_none(),
|
||||
"control: the singular agrees"
|
||||
);
|
||||
assert_eq!(all.len(), 4);
|
||||
}
|
||||
Reference in New Issue
Block a user