riix.metrics

 1"""module for computing metrics for rating system experiments"""
 2
 3import numpy as np
 4
 5
 6def binary_accuracy(probs: np.ndarray, outcomes: np.ndarray) -> float:
 7    """compute accuracy where outcomes is binary ties count for half"""
 8    pos_mask = probs > 0.5
 9    neg_mask = probs < 0.5
10    draw_mask = probs == 0.5
11    correct = outcomes[pos_mask].sum() + (1.0 - outcomes[neg_mask]).sum() + 0.5 * draw_mask.sum()
12    return correct / probs.shape[0]
13
14
15def accuracy_without_draws(probs: np.ndarray, outcomes: np.ndarray) -> float:
16    """compute binary accuracy after first filtering out rows where the label is a draw"""
17    draw_mask = outcomes == 0.5
18    probs = probs[~draw_mask]
19    outcomes = outcomes[~draw_mask]
20    return binary_accuracy(probs, outcomes)
21
22
23def accuracy_with_draws(probs: np.ndarray, outcomes: np.ndarray, draw_margin=0.0) -> float:
24    """computes accuracy where probs within draw_margin of 0.5 are treated as draw predictions"""
25    pos_pred_mask = probs > (0.5 + draw_margin)
26    neg_pred_mask = probs < (0.5 - draw_margin)
27    draw_pred_mask = np.abs(probs - 0.5) <= draw_margin
28    correct = outcomes[pos_pred_mask].sum()
29    correct += (1.0 - outcomes)[neg_pred_mask].sum()
30    correct += (outcomes[draw_pred_mask] == 0.5).sum()
31    return correct / outcomes.shape[0]
32
33
34def binary_log_loss(probs: np.ndarray, outcomes: np.ndarray, eps: float = 1e-6) -> float:
35    """compute log loss where outcome is binary 1.0 or 0.0"""
36    probs = np.clip(probs, eps, 1 - eps)
37    loss_array = -(np.log(probs) * outcomes) - (np.log(1.0 - probs) * (1.0 - outcomes))
38    return loss_array.mean()
39
40
41def brier_score(probs: np.ndarray, outcomes: np.ndarray) -> float:
42    """compute the brier score, which is equivalent to the MSE"""
43    return np.square(probs - outcomes).mean()
44
45
46def binary_metrics_suite(probs: np.ndarray, outcomes: np.ndarray):
47    """a wrapper class for running a bunch of binary metrics"""
48    metrics = {
49        'accuracy': float(binary_accuracy(probs, outcomes)),
50        'accuracy_without_draws': float(accuracy_without_draws(probs, outcomes)),
51        'log_loss': float(binary_log_loss(probs, outcomes)),
52        'brier_score': float(brier_score(probs, outcomes)),
53    }
54    return metrics