Approach (brief):- Compute per-user revenue, group by variant, estimate incremental revenue per user = mean(revenue_treatment) - mean(revenue_control). Use both a parametric t-test (if revenue roughly normal after transform or large N) and bootstrap (non-parametric, robust to skewed revenue). Report point estimate, 95% CI, and p-value.Python (bootstrap + t-test pseudocode):python
import pandas as pd
import numpy as np
from scipy import stats
df = pd.read_csv("experiments.csv")
control = df[df.variant=='control']['revenue'].values
treat = df[df.variant=='treatment']['revenue'].values
# point estimate
delta = treat.mean() - control.mean()
# parametric t-test (Welch)
tstat, pval = stats.ttest_ind(treat, control, equal_var=False)
# bootstrap CI
def bootstrap_diff(a,b,iterations=10000):
diffs = []
for _ in range(iterations):
s1 = np.random.choice(a, size=len(a), replace=True)
s2 = np.random.choice(b, size=len(b), replace=True)
diffs.append(s2.mean()-s1.mean())
return np.percentile(diffs, [2.5,97.5]), np.mean(diffs)
ci, boot_mean = bootstrap_diff(control, treat)
sql
WITH per_user AS (
SELECT user_id, variant, SUM(revenue) AS revenue
FROM events
GROUP BY user_id, variant
)
SELECT
AVG(CASE WHEN variant='treatment' THEN revenue END) AS mean_treat,
AVG(CASE WHEN variant='control' THEN revenue END) AS mean_control,
(AVG(CASE WHEN variant='treatment' THEN revenue END) -
AVG(CASE WHEN variant='control' THEN revenue END)) AS incremental_per_user
FROM per_user;
