Model Evaluation and Validation Questions

def roc_auc_trapz(y_true, y_score):
    """
    Compute ROC AUC via trapezoidal rule without sklearn.
    y_true: iterable of 0/1 labels
    y_score: iterable of floats (higher => more likely positive)
    Returns AUC in [0,1]
    """
    import math
    # Convert to lists
    y_true = list(y_true)
    y_score = list(y_score)
    n = len(y_true)
    if n == 0:
        raise ValueError("Empty input")
    # Count positives and negatives
    P = sum(1 for y in y_true if y == 1)
    N = n - P
    if P == 0 or N == 0:
        return float('nan')  # undefined AUC when only one class present

    # Sort by score descending, tie-stable
    paired = sorted(zip(y_score, y_true), key=lambda x: (-x[0], -x[1]))
    # Build TPR/FPR points by grouping ties
    tp = 0
    fp = 0
    points = [(0.0, 0.0)]  # (FPR, TPR)
    i = 0
    while i < n:
        score = paired[i][0]
        # process all with same score
        tp_inc = 0
        fp_inc = 0
        while i < n and paired[i][0] == score:
            if paired[i][1] == 1:
                tp_inc += 1
            else:
                fp_inc += 1
            i += 1
        tp += tp_inc
        fp += fp_inc
        points.append((fp / N, tp / P))

    # Ensure curve ends at (1,1)
    if points[-1] != (1.0, 1.0):
        points.append((1.0, 1.0))

    # Trapezoidal integration along FPR axis
    auc = 0.0
    for (fpr1, tpr1), (fpr2, tpr2) in zip(points, points[1:]):
        auc += (fpr2 - fpr1) * (tpr1 + tpr2) / 2.0
    # Clamp for tiny numerical errors
    return max(0.0, min(1.0, auc))

import random
from collections import defaultdict

def stratified_kfold_indices(y, k=5, seed=None):
    """
    y: list/array of class labels (length n)
    k: number of folds
    Returns: list of (train_idx, test_idx) pairs for each fold
    """
    if seed is not None:
        random.seed(seed)
    n = len(y)
    # Group indices by class
    cls_idx = defaultdict(list)
    for i, label in enumerate(y):
        cls_idx[label].append(i)
    # Shuffle indices within each class
    for lst in cls_idx.values():
        random.shuffle(lst)
    # Initialize empty lists for each fold
    folds = [[] for _ in range(k)]
    # Distribute indices of each class across folds round-robin
    for lst in cls_idx.values():
        for i, idx in enumerate(lst):
            folds[i % k].append(idx)
    # Build train/test pairs
    pairs = []
    for i in range(k):
        test_idx = sorted(folds[i])
        train_idx = sorted([idx for f in range(k) if f != i for idx in folds[f]])
        pairs.append((train_idx, test_idx))
    return pairs