A/B Test Results Analysis Questions

import math
from typing import Tuple

def two_prop_z_test(success_c: int, n_c: int, success_t: int, n_t: int, alpha: float = 0.05) -> Tuple[float, Tuple[float, float]]:
    """
    Returns (p_value_two_sided, (ci_lower, ci_upper)) for difference in proportions (treatment - control)
    using normal approximation.
    Raises ValueError for non-positive sample sizes.
    """
    if n_c <= 0 or n_t <= 0:
        raise ValueError("Sample sizes must be > 0.")
    # sample proportions
    p_c = success_c / n_c
    p_t = success_t / n_t
    diff = p_t - p_c

    # pooled proportion for hypothesis p_t == p_c (used in z for p-value)
    pooled = (success_c + success_t) / (n_c + n_t)
    se_pooled = math.sqrt(pooled * (1 - pooled) * (1/n_c + 1/n_t))

    # handle zero SE (happens when pooled is 0 or 1)
    if se_pooled == 0:
        # if pooled exactly 0 or 1, then proportions are identical at extremes => p-value 1 if diff==0 else 0
        p_value = 1.0 if abs(diff) < 1e-12 else 0.0
    else:
        z = diff / se_pooled
        # two-sided p-value
        p_value = 2 * (1 - 0.5*(1 + math.erf(abs(z)/math.sqrt(2))))

    # standard error for confidence interval (unpooled)
    se_ci = math.sqrt(p_t*(1-p_t)/n_t + p_c*(1-p_c)/n_c)
    # z critical for two-sided alpha (approx)
    z_crit = abs(math.sqrt(2) * math.erfinv(1 - alpha)) if hasattr(math, "erfinv") else 1.96
    # math.erfinv exists in Python 3.8+; fallback to 1.96 for 95%
    if 'erfinv' in dir(math):
        z_crit = math.sqrt(2) * math.erfinv(1 - alpha)

    ci_lower = diff - z_crit * se_ci
    ci_upper = diff + z_crit * se_ci

    return float(p_value), (float(ci_lower), float(ci_upper))

import numpy as np

def bootstrap_median_ci(revenues, n_boot=10000, ci=95, seed=None):
    """
    Compute a bootstrap confidence interval for the median.
    revenues: 1D array-like of per-user revenues
    n_boot: number of bootstrap iterations
    ci: percentile CI width (e.g., 95)
    seed: int or None for reproducibility
    Returns: (median_point_estimate, lower_bound, upper_bound)
    """
    x = np.asarray(revenues)
    if x.ndim != 1:
        raise ValueError("revenues must be a 1D array-like")
    x = x[~np.isnan(x)]  # drop NaNs
    n = x.size
    if n == 0:
        raise ValueError("no valid observations")
    rng = np.random.default_rng(seed)
    # sample indices in a (n_boot, n) matrix efficiently
    idx = rng.integers(0, n, size=(n_boot, n))
    samples = x[idx]                # shape (n_boot, n)
    medians = np.median(samples, axis=1)
    lower_p = (100 - ci) / 2
    upper_p = 100 - lower_p
    lo, hi = np.percentile(medians, [lower_p, upper_p])
    return np.median(x), lo, hi