Compare commits
2
Commits
6e2ce69728
...
56193609f7
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
56193609f7 | ||
|
|
13a395fdc9 |
+15
-7
@@ -171,13 +171,21 @@ where
|
||||
|
||||
/// Set explicit per-team continuous scores with a per-event noise override.
|
||||
///
|
||||
/// `sigma` overrides `HistoryBuilder::score_sigma` for this event only.
|
||||
/// Must be `> 0.0`. Constructing the outcome with a non-positive or NaN
|
||||
/// sigma is allowed; the value is rejected with
|
||||
/// `InferenceError::InvalidParameter` when the event is ingested, so
|
||||
/// callers get an error from `commit` rather than a panic.
|
||||
pub fn scores_with_sigma<I: IntoIterator<Item = f64>>(mut self, scores: I, sigma: f64) -> Self {
|
||||
self.event.outcome = crate::Outcome::scores_with_sigma(scores, sigma);
|
||||
/// `score_sigma` is the observation noise on the *score margin*, not a
|
||||
/// skill sigma, and it overrides `HistoryBuilder::score_sigma` for this
|
||||
/// event only. A small value takes the margin near-literally; a large one
|
||||
/// barely moves the ratings.
|
||||
///
|
||||
/// Must be `> 0.0`. Building the outcome with a non-positive or NaN value
|
||||
/// is allowed; it is rejected with `InferenceError::InvalidParameter` when
|
||||
/// the event is ingested, so callers get an error from `commit` rather
|
||||
/// than a panic.
|
||||
pub fn scores_with_noise<I: IntoIterator<Item = f64>>(
|
||||
mut self,
|
||||
scores: I,
|
||||
score_sigma: f64,
|
||||
) -> Self {
|
||||
self.event.outcome = crate::Outcome::scores_with_noise(scores, score_sigma);
|
||||
self
|
||||
}
|
||||
|
||||
|
||||
+20
-13
@@ -1188,7 +1188,7 @@ impl<T: Time, D: Drift<T>, O: Observer<T>, K: Eq + Hash + Clone> History<T, D, O
|
||||
// `converge` refuses to report a NaN fit, but nothing stopped a
|
||||
// caller ignoring that error and predicting anyway. Measured on
|
||||
// a point-mass-prior history with `beta(0.0)`, after `converge`
|
||||
// returned `NonFiniteResult`: `predict_quality` gave `Ok(NaN)`,
|
||||
// returned `NonFiniteResult`: `quality` gave `Ok(NaN)`,
|
||||
// `predict_outcome().total()` gave `NaN`, and
|
||||
// `predict_win_probabilities` gave `Ok([0.0, 0.0])` — finite,
|
||||
// plausible, and summing to zero against a doc that promises
|
||||
@@ -1226,7 +1226,7 @@ impl<T: Time, D: Drift<T>, O: Observer<T>, K: Eq + Hash + Clone> History<T, D, O
|
||||
//
|
||||
// Every prediction here is a statement about how performances *vary*,
|
||||
// and in this configuration nothing varies. The consequences were three
|
||||
// different wrong answers rather than one error. `predict_quality`
|
||||
// different wrong answers rather than one error. `quality`
|
||||
// **panicked** — "cannot invert a singular matrix", from a
|
||||
// `Result`-returning method, on a history that had converged cleanly —
|
||||
// because the contrast covariance `beta^2 A^T A + A^T S A` is exactly
|
||||
@@ -1301,14 +1301,21 @@ impl<T: Time, D: Drift<T>, O: Observer<T>, K: Eq + Hash + Clone> History<T, D, O
|
||||
})
|
||||
}
|
||||
|
||||
/// Draw-probability quality metric for the given teams (key slices).
|
||||
/// How fair a matchup between these teams would be, against the fit.
|
||||
///
|
||||
/// Values range roughly `[0, 1]`; 1 == perfectly matched. Supports any
|
||||
/// Values range roughly `[0, 1]`; 1 is perfectly matched. Supports any
|
||||
/// number of teams.
|
||||
///
|
||||
/// Note this answers "is this matchup *fair*", which is not the same as
|
||||
/// "is this matchup *informative*" — the two coincide for two evenly
|
||||
/// matched teams and diverge elsewhere.
|
||||
/// The method form of the free [`quality`](crate::quality), which scores a
|
||||
/// hypothetical from explicit skill distributions instead. That is the rule
|
||||
/// the whole family follows: a free function takes parameters, the
|
||||
/// same-named `History` method asks the question against what was fitted.
|
||||
///
|
||||
/// It was `predict_quality` until #78 pointed out that it predicts nothing
|
||||
/// — it answers "is this matchup *fair*", not "what will happen". Fair is
|
||||
/// also not the same as *informative*: the two coincide for two evenly
|
||||
/// matched teams and diverge elsewhere. See
|
||||
/// [`History::expected_information_gain`] for the other question.
|
||||
///
|
||||
/// # Preconditions
|
||||
///
|
||||
@@ -1329,7 +1336,7 @@ impl<T: Time, D: Drift<T>, O: Observer<T>, K: Eq + Hash + Clone> History<T, D, O
|
||||
/// — the fit did not converge — and `InvalidParameter` if `beta` is zero
|
||||
/// and every skill is a point mass, leaving no performance distribution to
|
||||
/// predict from.
|
||||
pub fn predict_quality<Q>(&self, teams: &[&[&Q]]) -> Result<f64, InferenceError>
|
||||
pub fn quality<Q>(&self, teams: &[&[&Q]]) -> Result<f64, InferenceError>
|
||||
where
|
||||
K: Borrow<Q>,
|
||||
Q: Hash + Eq + ?Sized + std::fmt::Debug,
|
||||
@@ -1826,7 +1833,7 @@ impl<T: Time, D: Drift<T>, O: Observer<T>, K: Eq + Hash + Clone> History<T, D, O
|
||||
/// ones that would actually be fitted if the matchup were played and
|
||||
/// recorded.
|
||||
///
|
||||
/// Distinct from [`History::predict_quality`], which measures *fairness*.
|
||||
/// Distinct from [`History::quality`], which measures *fairness*.
|
||||
/// The two coincide for two evenly matched competitors and diverge
|
||||
/// elsewhere. See [`expected_information_gain`](crate::expected_information_gain)
|
||||
/// for the scale, the analytic `ln k` ceiling, and the cost.
|
||||
@@ -4153,7 +4160,7 @@ mod tests {
|
||||
crate::Team::with_members([crate::Member::new("a")]),
|
||||
crate::Team::with_members([crate::Member::new("b")]),
|
||||
],
|
||||
outcome: Outcome::scores_with_sigma([3.0, 1.0], 0.5),
|
||||
outcome: Outcome::scores_with_noise([3.0, 1.0], 0.5),
|
||||
}])
|
||||
.unwrap();
|
||||
let _ = h_a.converge().unwrap();
|
||||
@@ -4195,7 +4202,7 @@ mod tests {
|
||||
crate::Team::with_members([crate::Member::new("a")]),
|
||||
crate::Team::with_members([crate::Member::new("b")]),
|
||||
],
|
||||
outcome: Outcome::scores_with_sigma([3.0, 1.0], 2.0),
|
||||
outcome: Outcome::scores_with_noise([3.0, 1.0], 2.0),
|
||||
}])
|
||||
.unwrap();
|
||||
let _ = h_a.converge().unwrap();
|
||||
@@ -4261,7 +4268,7 @@ mod tests {
|
||||
h_a.event(0_i64)
|
||||
.team(["a"])
|
||||
.team(["b"])
|
||||
.scores_with_sigma([3.0, 1.0], 2.0)
|
||||
.scores_with_noise([3.0, 1.0], 2.0)
|
||||
.commit()
|
||||
.unwrap();
|
||||
let _ = h_a.converge().unwrap();
|
||||
@@ -4274,7 +4281,7 @@ mod tests {
|
||||
crate::Team::with_members([crate::Member::new("a")]),
|
||||
crate::Team::with_members([crate::Member::new("b")]),
|
||||
],
|
||||
outcome: Outcome::scores_with_sigma([3.0, 1.0], 2.0),
|
||||
outcome: Outcome::scores_with_noise([3.0, 1.0], 2.0),
|
||||
}])
|
||||
.unwrap();
|
||||
let _ = h_b.converge().unwrap();
|
||||
|
||||
+9
-4
@@ -113,11 +113,16 @@ impl Outcome {
|
||||
|
||||
/// Explicit per-team continuous scores with a per-event noise override.
|
||||
///
|
||||
/// The noise is on the *observed score margin*, in the units of the scores
|
||||
/// themselves — it is not a skill sigma, which is what the old name
|
||||
/// `scores_with_sigma` read as. It overrides `HistoryBuilder::score_sigma`
|
||||
/// for this event only.
|
||||
///
|
||||
/// `score_sigma` must be `> 0.0`. Constructing an `Outcome` with a
|
||||
/// non-positive or NaN value is allowed; the value is rejected with
|
||||
/// `InferenceError::InvalidParameter` when the event is ingested, so
|
||||
/// callers get an error rather than a panic.
|
||||
pub fn scores_with_sigma<I: IntoIterator<Item = f64>>(scores: I, score_sigma: f64) -> Self {
|
||||
pub fn scores_with_noise<I: IntoIterator<Item = f64>>(scores: I, score_sigma: f64) -> Self {
|
||||
Self::Scored {
|
||||
scores: scores.into_iter().collect(),
|
||||
score_sigma: Some(score_sigma),
|
||||
@@ -211,7 +216,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn scores_with_sigma_round_trips() {
|
||||
let o = Outcome::scores_with_sigma([10.0, 4.0], 0.5);
|
||||
let o = Outcome::scores_with_noise([10.0, 4.0], 0.5);
|
||||
assert_eq!(o.team_count(), 2);
|
||||
assert_eq!(o.as_scores(), Some(&[10.0, 4.0][..]));
|
||||
}
|
||||
@@ -227,7 +232,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn scores_with_sigma_sets_sigma_some() {
|
||||
let o = Outcome::scores_with_sigma([3.0, 1.0], 2.0);
|
||||
let o = Outcome::scores_with_noise([3.0, 1.0], 2.0);
|
||||
match o {
|
||||
Outcome::Scored { score_sigma, .. } => assert_eq!(score_sigma, Some(2.0)),
|
||||
Outcome::Ranked(_) => panic!("expected Scored variant"),
|
||||
@@ -239,7 +244,7 @@ mod tests {
|
||||
/// `tests/degenerate_inputs.rs::scored_event_rejects_non_positive_sigma`.
|
||||
#[test]
|
||||
fn scores_with_sigma_defers_validation_to_ingestion() {
|
||||
let o = Outcome::scores_with_sigma([3.0, 1.0], 0.0);
|
||||
let o = Outcome::scores_with_noise([3.0, 1.0], 0.0);
|
||||
match o {
|
||||
Outcome::Scored { score_sigma, .. } => assert_eq!(score_sigma, Some(0.0)),
|
||||
Outcome::Ranked(_) => panic!("expected Scored variant"),
|
||||
|
||||
+1
-1
@@ -203,7 +203,7 @@ fn predict_quality_two_teams() {
|
||||
h.record_winner(&"a", &"b", 1).unwrap();
|
||||
let _ = h.converge().unwrap();
|
||||
|
||||
let q = h.predict_quality(&[&[&"a"], &[&"b"]]).unwrap();
|
||||
let q = h.quality(&[&[&"a"], &[&"b"]]).unwrap();
|
||||
assert!(q > 0.0 && q <= 1.0);
|
||||
}
|
||||
|
||||
|
||||
@@ -102,7 +102,7 @@ fn every_magnitude_parameter_rejects_a_negative_value() {
|
||||
}),
|
||||
),
|
||||
(
|
||||
"Outcome::scores_with_sigma (at ingestion)",
|
||||
"Outcome::scores_with_noise (at ingestion)",
|
||||
Box::new(|v| {
|
||||
let mut h = History::builder().build();
|
||||
h.add_events(vec![trueskill_tt::Event {
|
||||
@@ -111,7 +111,7 @@ fn every_magnitude_parameter_rejects_a_negative_value() {
|
||||
trueskill_tt::Team::with_members([Member::new("a")]),
|
||||
trueskill_tt::Team::with_members([Member::new("b")]),
|
||||
],
|
||||
outcome: Outcome::scores_with_sigma([3.0, 1.0], v),
|
||||
outcome: Outcome::scores_with_noise([3.0, 1.0], v),
|
||||
}])
|
||||
.is_err()
|
||||
}),
|
||||
|
||||
@@ -222,7 +222,7 @@ fn scored_event_rejects_non_positive_sigma() {
|
||||
.event(1)
|
||||
.team(["a"])
|
||||
.team(["b"])
|
||||
.scores_with_sigma([3.0, 1.0], f64::NAN)
|
||||
.scores_with_noise([3.0, 1.0], f64::NAN)
|
||||
.commit()
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
|
||||
@@ -52,7 +52,7 @@ fn every_team_shaped_query_accepts_the_same_slice() {
|
||||
let h = owned();
|
||||
let teams: &[&[&str]] = &[&["alice"], &["bob"]];
|
||||
|
||||
h.predict_quality(teams).expect("quality");
|
||||
h.quality(teams).expect("quality");
|
||||
let _ = h.predict_outcome(teams).expect("outcome");
|
||||
h.predict_ranking(teams, &[0, 1]).expect("ranking");
|
||||
h.expected_information_gain(teams)
|
||||
|
||||
+2
-2
@@ -34,7 +34,7 @@ fn unknown_keys_are_reported_not_silently_dropped() {
|
||||
h.predict_win_probabilities(&[&[&"a"], &[&"ghost"]])
|
||||
.is_err()
|
||||
);
|
||||
assert!(h.predict_quality(&[&[&"a"], &[&"ghost"]]).is_err());
|
||||
assert!(h.quality(&[&[&"a"], &[&"ghost"]]).is_err());
|
||||
assert!(h.predict_ranking(&[&[&"a"], &[&"ghost"]], &[0, 1]).is_err());
|
||||
}
|
||||
|
||||
@@ -407,7 +407,7 @@ fn prior_reaches_every_prediction_entry_point() {
|
||||
let h = history_with_policy(&["a", "b"], trueskill_tt::UnknownKeys::Prior);
|
||||
let teams: &[&[&&str]] = &[&[&"a"], &[&"ghost"]];
|
||||
|
||||
assert!(h.predict_quality(teams).is_ok());
|
||||
assert!(h.quality(teams).is_ok());
|
||||
assert!(h.predict_win_probabilities(teams).is_ok());
|
||||
assert!(h.predict_outcome(teams).is_ok());
|
||||
assert!(h.predict_ranking(teams, &[0, 1]).is_ok());
|
||||
|
||||
@@ -78,7 +78,7 @@ macro_rules! all_predictions {
|
||||
($h:ident, $f:expr) => {{
|
||||
let teams: &[&[&&'static str]] = &[&[&"a"], &[&"b"]];
|
||||
let f = $f;
|
||||
f("predict_quality", $h.predict_quality(teams).map(|_| ()));
|
||||
f("quality", $h.quality(teams).map(|_| ()));
|
||||
f(
|
||||
"predict_win_probabilities",
|
||||
$h.predict_win_probabilities(teams).map(|_| ()),
|
||||
@@ -116,7 +116,7 @@ fn degenerate_performances_are_refused_rather_than_answered_wrongly() {
|
||||
assert_eq!(skill.sigma(), 0.0);
|
||||
assert!(skill.mu().is_finite());
|
||||
|
||||
// `predict_quality` previously PANICKED here, out of a method that returns
|
||||
// `quality` previously PANICKED here, out of a method that returns
|
||||
// `Result`: the contrast covariance is exactly singular when beta is zero
|
||||
// and every skill is a point mass.
|
||||
all_predictions!(h, |name: &str, r: Result<(), InferenceError>| {
|
||||
|
||||
+2
-5
@@ -110,11 +110,8 @@ fn history_predict_quality_supports_three_teams() {
|
||||
h.record_winner(&"b", &"c", 2).unwrap();
|
||||
let _ = h.converge().unwrap();
|
||||
|
||||
let q = h.predict_quality(&[&[&"a"], &[&"b"], &[&"c"]]).unwrap();
|
||||
assert!(
|
||||
q.is_finite(),
|
||||
"3-team predict_quality must be finite, got {q}"
|
||||
);
|
||||
let q = h.quality(&[&[&"a"], &[&"b"], &[&"c"]]).unwrap();
|
||||
assert!(q.is_finite(), "3-team quality must be finite, got {q}");
|
||||
assert!((0.0..=1.0).contains(&q), "out of range: {q}");
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -139,7 +139,7 @@ fn ingestion_rejects_a_tie_without_a_draw_probability() {
|
||||
);
|
||||
}
|
||||
|
||||
/// `Outcome::scores_with_sigma` documents that a non-positive sigma is
|
||||
/// `Outcome::scores_with_noise` documents that a non-positive sigma is
|
||||
/// accepted at construction and rejected at ingestion.
|
||||
#[test]
|
||||
fn ingestion_rejects_a_non_positive_per_event_score_sigma() {
|
||||
@@ -152,7 +152,7 @@ fn ingestion_rejects_a_non_positive_per_event_score_sigma() {
|
||||
Team::with_members([Member::new("a")]),
|
||||
Team::with_members([Member::new("b")]),
|
||||
],
|
||||
outcome: Outcome::scores_with_sigma([21.0, 9.0], sigma),
|
||||
outcome: Outcome::scores_with_noise([21.0, 9.0], sigma),
|
||||
}])
|
||||
.expect_err("a non-positive per-event sigma must be rejected");
|
||||
assert!(
|
||||
|
||||
Reference in New Issue
Block a user