To estimate expected cost for a spam filter and find optimal thresholds under varying prevalence and costs, we can simulate expected cost using classifier scores (probabilities). For each threshold, compute predicted labels, then expected cost = FP_count * fp_cost + FN_count * fn_cost, normalized per message or left as total. We can sweep thresholds and prevalences to see sensitivity.Approach:1. Use classifier_scores and true labels sampled according to prevalence to simulate many messages.2. For each threshold, compute FP/FN and cost.3. Return costs and optimal threshold(s). Show how varying prevalence and costs shifts optima.python
import numpy as np
def estimate_spam_cost(prevalence, fp_cost, fn_cost, classifier_scores, threshold):
"""
classifier_scores: array-like of classifier p(spam) for a set of examples
prevalence: P(true=spam) in [0,1]
fp_cost: cost of false positive (blocking legit mail)
fn_cost: cost of false negative (letting spam through)
threshold: decision threshold for predicting spam (>= -> spam)
Returns expected cost per example.
"""
scores = np.asarray(classifier_scores)
n = len(scores)
# Simulate true labels according to prevalence
rng = np.random.default_rng(0)
true = rng.random(n) < prevalence
preds = scores >= threshold
fp = np.sum((preds == 1) & (true == 0))
fn = np.sum((preds == 0) & (true == 1))
# expected cost per message
return (fp * fp_cost + fn * fn_cost) / n
def sweep_thresholds(prevalence, fp_cost, fn_cost, classifier_scores, n_thresholds=101):
ths = np.linspace(0,1,n_thresholds)
costs = [estimate_spam_cost(prevalence, fp_cost, fn_cost, classifier_scores, t) for t in ths]
best_idx = int(np.argmin(costs))
return ths, np.array(costs), ths[best_idx], costs[best_idx]
# Sample demonstration
if __name__ == "__main__":
# synthetic classifier scores (higher -> more likely spam)
rng = np.random.default_rng(42)
# mixture: legit ~ Beta(2,8), spam ~ Beta(8,2)
legit = rng.beta(2,8,10000)
spam = rng.beta(8,2,10000)
scores = np.concatenate([legit, spam])
for prevalence in [0.01, 0.1, 0.5]:
for fp_cost, fn_cost in [(1,5),(1,20),(5,1)]:
ths, costs, best_t, best_cost = sweep_thresholds(prevalence, fp_cost, fn_cost, scores)
print(f"prev={prevalence:.2f} fp={fp_cost} fn={fn_cost} -> best_th={best_t:.2f} cost={best_cost:.4f}")
