Experimentation Metrics and Strategy Questions

import numpy as np
def simulate_one(p0, p1, n_per_look, bounds):
    total_n = 0
    for look, n in enumerate(n_per_look):
        total_n = n
        # simulate data up to current look
        x_ctrl = np.random.binomial(n//2, p0)
        x_treat = np.random.binomial(n//2, p1)
        # compute z-stat
        p_pool = (x_ctrl + x_treat) / n
        se = np.sqrt(p_pool*(1-p_pool)*(2/(n//2)))
        z = (x_treat/(n//2) - x_ctrl/(n//2)) / se
        if z >= bounds['upper'][look]:
            return 'efficacy', look+1
        if 'lower' in bounds and z <= bounds['lower'][look]:
            return 'futility', look+1
    return 'none', len(n_per_look)

-- build account-level outcomes and treatment indicator
WITH account_agg AS (
  SELECT
    a.account_id,
    MAX(a.treatment) AS treatment,         -- treatment assigned at account
    COUNT(DISTINCT u.user_id) AS n_users,
    SUM(e.revenue) AS total_revenue,
    SUM(CASE WHEN e.event_type='purchase' THEN 1 ELSE 0 END) AS purchases,
    AVG(u.signup_age_days) AS avg_signup_age
  FROM accounts a
  LEFT JOIN users u ON u.account_id = a.account_id
  LEFT JOIN events e ON e.user_id = u.user_id
  GROUP BY a.account_id
)
SELECT * FROM account_agg;

-- After computing beta_hat (from external routine), compute residuals per row
WITH resid AS (
  SELECT
    a.account_id,
    u.user_id,
    y.metric AS y,
    t.treatment AS t,
    -- other covariates x1,x2...
    (y - (b0 + b1*t + b2*x1 + ...)) AS resid
  FROM users u
  JOIN accounts a ON a.account_id = u.account_id
  JOIN ... -- join metrics
)
-- compute per-cluster score sums
SELECT
  account_id,
  SUM(resid * 1) AS S_intercept,
  SUM(resid * treatment) AS S_treatment,
  SUM(resid * x1) AS S_x1
FROM resid
GROUP BY account_id;