Experiment Prioritization & Roadmap Questions

import numpy as np

def benjamini_hochberg(p_values, q):
    """
    Benjamini-Hochberg FDR control.
    p_values: list or 1D numpy array of p-values (may contain NaN)
    q: target FDR level (0 < q < 1)
    Returns: boolean numpy array (same shape) where True = reject null
    """
    p = np.asarray(p_values)
    if p.ndim != 1:
        raise ValueError("p_values must be a 1D array or list")
    if not (0 < q < 1):
        raise ValueError("q must be between 0 and 1")

    n = len(p)
    mask_nan = np.isnan(p)
    # Replace NaNs with +inf so they never get rejected; keep track to restore
    p_for_sort = p.copy()
    p_for_sort[mask_nan] = np.inf

    order = np.argsort(p_for_sort)           # indices of sorted p (ascending)
    sorted_p = p_for_sort[order]
    ranks = np.arange(1, n + 1)             # 1-based ranks

    thresholds = ranks / n * q
    # Find largest k where sorted_p[k-1] <= thresholds[k-1]
    valid = sorted_p <= thresholds
    if not np.any(valid):
        return np.zeros(n, dtype=bool)
    k = np.max(np.where(valid)[0])          # index (0-based) of largest valid
    cutoff = sorted_p[k]

    # Reject all original p <= cutoff (and not NaN)
    rejects = (p <= cutoff) & (~mask_nan)
    return rejects

import math
import numpy as np
from typing import Iterator, Tuple, Dict

def pocock_spent_alpha(window_idx: int, max_windows: int, alpha: float) -> float:
    # Pocock-like spending: monotone concave spending (approximation via log)
    t = min(window_idx, max_windows)
    return alpha * math.log(1 + (math.e - 1) * t / max_windows) / math.log(math.e)

def two_sample_z_pvalue(c_ev, c_tr, t_ev, t_tr) -> float:
    # pooled proportion under H0
    p_pool = (c_ev + t_ev) / (c_tr + t_tr) if (c_tr + t_tr) > 0 else 0.0
    se = math.sqrt(p_pool * (1 - p_pool) * (1.0 / c_tr + 1.0 / t_tr))
    if se == 0:
        return 1.0
    diff = (t_ev / t_tr) - (c_ev / c_tr)
    z = diff / se
    # two-sided p-value
    return 2 * (1 - 0.5 * (1 + math.erf(abs(z) / math.sqrt(2))))

def evaluate_stream(stream_iter: Iterator[Tuple[int,int,int,int]],
                    alpha: float = 0.05,
                    min_samples: int = 100,
                    max_windows: int = 1000) -> Dict:
    window = 0
    for (c_ev, c_tr, t_ev, t_tr) in stream_iter:
        window += 1
        if window > max_windows:
            break
        total_samples = c_tr + t_tr
        p_val = two_sample_z_pvalue(c_ev, c_tr, t_ev, t_tr)
        if total_samples < min_samples:
            decision = "insufficient_samples"
            # continue to next look but we may still report current p-value
        else:
            alpha_spent = pocock_spent_alpha(window, max_windows, alpha)
            # convert cumulative spent alpha to a per-look threshold: using cumulative spending,
            # we require p_val <= alpha_spent to reject at this look
            if p_val <= alpha_spent:
                return {"decision": "reject", "p_value": p_val, "window_index": window}
            decision = "continue"
    return {"decision": decision, "p_value": p_val if 'p_val' in locals() else None, "window_index": window}