Customer Obsession Questions

# Inputs:
# experiments: list of experiments; each has variants A,B with metrics mean_A, mean_B, var_A, var_B, n_A, n_B
# overlap_matrix: matrix where overlap_matrix[i][j] = fraction of samples shared between exp i and j (0..1)
# alpha, m = desired alpha, number_of_comparisons
# looks = list of interim look fractions (e.g., [0.25,0.5,0.75,1.0])

function compute_results(experiments, overlap_matrix, alpha, m, looks):
  k = len(experiments)
  results = []
  # per-experiment uplift and z
  for i,exp in enumerate(experiments):
    uplift = exp.mean_B - exp.mean_A
    # pooled variance for two-sample approx
    se = sqrt( exp.var_A/exp.n_A + exp.var_B/exp.n_B )
    z = uplift / se
    p = two_tailed_p_from_z(z)
    results.append({i, uplift, z, p, se})

  # build covariance matrix of z-statistics due to overlap
  cov = identity_matrix(k)
  for i in range(k):
    for j in range(i+1,k):
      # approximate covariance via overlap fraction and shared variance
      f = overlap_matrix[i][j]
      # normalize by standard errors to get correlation
      corr = f * sqrt( (experiments[i].var_A/experiments[i].n_A + experiments[i].var_B/experiments[i].n_B) *
                       (experiments[j].var_A/experiments[j].n_A + experiments[j].var_B/experiments[j].n_B) ) / (results[i].se * results[j].se)
      corr = clamp(corr, -0.999,0.999)
      cov[i][j] = cov[j][i] = corr

  # optional multivariate adjustment: compute effective number of independent tests (Meff)
  eigenvals = eigenvalues(cov)
  Meff = sum( eigenvals > 1e-6 )  # simple effective count

  # FDR control using Benjamini-Hochberg on raw p-values, but use m = Meff or provided m
  p_list = [r.p for r in results]
  bh_thresholds = sorted_pvalues_with_indices(p_list)
  # compute BH adjusted decisions
  discoveries = benjamini_hochberg(p_list, alpha, m= max(1,Meff))

  return {results, cov, Meff, discoveries}

# Sequential analysis: alpha-spending for interim looks
function alpha_spending_total(alpha, looks, method="obrien_fleming"):
  G = len(looks)
  alphas = []
  for g, frac in enumerate(looks, start=1):
    if method=="obrien_fleming":
      # approximate boundary: spend small alpha early
      boundary = normal_quantile(1 - alpha/2 / sqrt(G/g))
      spent = 2*(1 - normal_cdf(boundary))
    else: # pocock
      spent = alpha * (g / G)
    alphas.append(spent)
  # convert to incremental alphas per look
  incremental = [alphas[0]] + [alphas[i]-alphas[i-1] for i in range(1,G)]
  return incremental

# Use per-look alpha to threshold p-values (or apply BH with adjusted alpha per look)