feat!: name the unknown key, expose tail probabilities, flag short fits
Three issues from two downstream consumers, all small, all sharing a theme: the crate had the information and would not hand it over. #44 — `UnknownKey { team: 0, member: 0 }` did not say which key. A consumer upgrading 0.1.2 -> 0.4.1 had every one of 5591 predictions return this error, fell back to a neutral 0.5, and lost its entire metadata model for a day. Nothing crashed and nothing logged; it was found by sweeping an unrelated parameter and noticing the output did not move. The 0.4.0 change that made unknown keys an error was right — the error was just too anonymous to act on. It now carries the key's `Debug` rendering, and its `Display` says what to do about it. The precondition is documented on every prediction entry point, which the reporter said would alone have saved the day. #43 — `cdf` was `pub(crate)`, so a consumer asking "is this competitor below the cutoff" approximated it with a `mu + z * sigma` band and had no way to say what confidence any `z` bought. Adds `Gaussian::probability_below` / `probability_above`. The second is separate on purpose: `1 - cdf` collapses to exactly zero past ~8.3 sigma, and a stopping rule is evaluated precisely there. Both route through the survival function added in 0.4.1, so this is visibility rather than new numerics. #50 — `ConvergenceReport` was not `#[must_use]`, so the one signal that a fit stopped short was trivially discarded. It now is, and that immediately found 78 sites doing exactly that — including this crate's own ATP example, which was capped at 10 sweeps when the history needs 30. The example now reads the report and says so. `ITERATIONS = 30` is documented as the floor it is, with the three measurements to hand: 400 events over 100 competitors already stops there at ~7e-3 against a 1e-6 tolerance, the ATP example needs 30 at a much looser one, and a consumer's 2000-node model needs 76 to 161. BREAKING CHANGE: `InferenceError::UnknownKey` gains a `key` field, and the prediction methods now require `K: Debug` in order to fill it. Closes #43, #50. Refs #44 — its third ask, an opt-in `UnknownKeys::Skip` mode, is a live API question and deliberately not answered here. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011hcFjNDmHXZF8URGLku5zZ
This commit is contained in:
+55
-7
@@ -9,7 +9,7 @@ fn history_with(names: &[&'static str], p_draw: f64) -> History {
|
||||
for pair in names.windows(2) {
|
||||
h.record_winner(&pair[0], &pair[1], 1).unwrap();
|
||||
}
|
||||
h.converge().unwrap();
|
||||
let _ = h.converge().unwrap();
|
||||
h
|
||||
}
|
||||
|
||||
@@ -20,7 +20,14 @@ fn unknown_keys_are_reported_not_silently_dropped() {
|
||||
let err = h
|
||||
.predict_outcome(&[&[&"a"], &[&"ghost"]])
|
||||
.expect_err("an unknown key must not yield a confident prediction");
|
||||
assert_eq!(err, InferenceError::UnknownKey { team: 1, member: 0 });
|
||||
assert_eq!(
|
||||
err,
|
||||
InferenceError::UnknownKey {
|
||||
team: 1,
|
||||
member: 0,
|
||||
key: "\"ghost\"".to_owned(),
|
||||
}
|
||||
);
|
||||
|
||||
// Every prediction entry point, not just one.
|
||||
assert!(
|
||||
@@ -35,7 +42,14 @@ fn unknown_keys_are_reported_not_silently_dropped() {
|
||||
fn an_entirely_unknown_team_is_an_error() {
|
||||
let h = history_with(&["a", "b"], 0.0);
|
||||
let err = h.predict_outcome(&[&[&"a"], &[&"x", &"y"]]).unwrap_err();
|
||||
assert_eq!(err, InferenceError::UnknownKey { team: 1, member: 0 });
|
||||
assert_eq!(
|
||||
err,
|
||||
InferenceError::UnknownKey {
|
||||
team: 1,
|
||||
member: 0,
|
||||
key: "\"x\"".to_owned(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -184,7 +198,7 @@ fn the_stronger_competitor_is_favoured() {
|
||||
for t in 1..=10 {
|
||||
h.record_winner(&"strong", &"weak", t).unwrap();
|
||||
}
|
||||
h.converge().unwrap();
|
||||
let _ = h.converge().unwrap();
|
||||
|
||||
let p = h.predict_outcome(&[&[&"strong"], &[&"weak"]]).unwrap();
|
||||
let (best, _) = p.most_likely().expect("a most likely outcome");
|
||||
@@ -206,7 +220,7 @@ fn team_size_affects_the_prediction() {
|
||||
.winner(0)
|
||||
.commit()
|
||||
.unwrap();
|
||||
h.converge().unwrap();
|
||||
let _ = h.converge().unwrap();
|
||||
|
||||
let p = h.predict_outcome(&[&[&"a", &"b"], &[&"c"]]).unwrap();
|
||||
assert!((p.total() - 1.0).abs() < 1e-6, "total = {}", p.total());
|
||||
@@ -229,7 +243,7 @@ fn information_gain_prefers_the_uncertain_pairing() {
|
||||
h.record_winner(&"rival", &"known", t + 100).unwrap();
|
||||
}
|
||||
h.record_winner(&"known", &"newcomer", 500).unwrap();
|
||||
h.converge().unwrap();
|
||||
let _ = h.converge().unwrap();
|
||||
|
||||
let settled = h
|
||||
.expected_information_gain(&[&[&"known"], &[&"rival"]])
|
||||
@@ -271,7 +285,11 @@ fn information_gain_reports_unknown_keys() {
|
||||
assert_eq!(
|
||||
h.expected_information_gain(&[&[&"a"], &[&"ghost"]])
|
||||
.unwrap_err(),
|
||||
InferenceError::UnknownKey { team: 1, member: 0 }
|
||||
InferenceError::UnknownKey {
|
||||
team: 1,
|
||||
member: 0,
|
||||
key: "\"ghost\"".to_owned(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -289,3 +307,33 @@ fn information_gain_accounts_for_draws() {
|
||||
let dist = with_draws.predict_outcome(&[&[&"a"], &[&"b"]]).unwrap();
|
||||
assert!(dist.probability_of(&[0, 0]) > 0.0);
|
||||
}
|
||||
|
||||
/// The defect that cost a consumer a day: `UnknownKey { team: 0, member: 0 }`
|
||||
/// says nothing about *which* key is unknown, so the natural handling — log it,
|
||||
/// fall back to a neutral value — converts a total miss into a plausible
|
||||
/// constant. The key has to be in the error, and in its `Display`.
|
||||
#[test]
|
||||
fn unknown_key_names_the_key_it_could_not_find() {
|
||||
let h = history_with(&["a", "b"], 0.0);
|
||||
let err = h.predict_outcome(&[&[&"a"], &[&"never_seen"]]).unwrap_err();
|
||||
|
||||
match &err {
|
||||
InferenceError::UnknownKey { key, .. } => {
|
||||
assert!(
|
||||
key.contains("never_seen"),
|
||||
"the error should name the key, got {key}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected UnknownKey, got {other:?}"),
|
||||
}
|
||||
|
||||
let rendered = err.to_string();
|
||||
assert!(
|
||||
rendered.contains("never_seen"),
|
||||
"Display should name the key: {rendered}"
|
||||
);
|
||||
assert!(
|
||||
rendered.contains("pre-filter"),
|
||||
"Display should say what to do about it: {rendered}"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user