ML System Evaluation and Metrics Questions

import numpy as np
from sklearn.metrics import roc_auc_score

def auc_bootstrap_ci(y_true, y_pred, fold_ids=None, B=2000, alpha=0.05, stratified=True, cluster_by_fold=False, random_state=None):
    """
    Compute bootstrap CI for AUC.
    - y_true, y_pred: arrays aligned per-observation (CV predicted prob for each obs)
    - fold_ids: required if cluster_by_fold=True
    - B: bootstrap samples
    - stratified: when True and cluster_by_fold=False, resample within classes
    - cluster_by_fold: when True, resample folds (fold_ids required)
    Returns: (auc_point_estimate, lower, upper, all_bootstrap_aucs)
    """
    rng = np.random.default_rng(random_state)
    y_true = np.asarray(y_true)
    y_pred = np.asarray(y_pred)
    n = len(y_true)
    # point estimate
    auc0 = roc_auc_score(y_true, y_pred)
    boot_aucs = []
    if cluster_by_fold:
        if fold_ids is None:
            raise ValueError("fold_ids required for cluster bootstrap")
        folds = np.unique(fold_ids)
        fold_to_idx = {f: np.where(fold_ids == f)[0] for f in folds}
        for _ in range(B):
            sampled_folds = rng.choice(folds, size=len(folds), replace=True)
            idx = np.concatenate([fold_to_idx[f] for f in sampled_folds])
            # allow duplicates -> keep duplicated indices to reflect sampling
            boot_aucs.append(roc_auc_score(y_true[idx], y_pred[idx]))
    else:
        pos_idx = np.where(y_true == 1)[0]
        neg_idx = np.where(y_true == 0)[0]
        for _ in range(B):
            if stratified:
                s_pos = rng.choice(pos_idx, size=pos_idx.size, replace=True)
                s_neg = rng.choice(neg_idx, size=neg_idx.size, replace=True)
                idx = np.concatenate([s_pos, s_neg])
            else:
                idx = rng.choice(np.arange(n), size=n, replace=True)
            boot_aucs.append(roc_auc_score(y_true[idx], y_pred[idx]))
    boot_aucs = np.array(boot_aucs)
    lower = np.percentile(boot_aucs, 100 * (alpha/2))
    upper = np.percentile(boot_aucs, 100 * (1 - alpha/2))
    return auc0, lower, upper, boot_aucs