Problem Solving and Analytical Thinking Questions

import pandas as pd
import numpy as np

def flag_anomalous_users(df, user_col='user_id', time_col='timestamp',
                         pred_col='pred', label_col='label',
                         short_window=7, hist_lookback=90, shrink_alpha=20):
    # df: timestamped rows per user with predictions and ground-truth labels
    df = df.copy()
    df[time_col] = pd.to_datetime(df[time_col])
    df = df.sort_values([user_col, time_col])

    # compute per-row absolute error
    df['abs_err'] = (df[pred_col] - df[label_col]).abs()

    # rolling short-window MAE per user (current)
    df['rolling_mae'] = df.groupby(user_col)['abs_err']\
                         .transform(lambda s: s.rolling(window=short_window, min_periods=1).mean())

    # For historical stats, compute rolling aggregates over a longer lookback but shift so history excludes the short window
    def hist_stats(s):
        # rolling over hist_lookback, then shift by short_window to exclude recent period
        roll = s.rolling(window=hist_lookback, min_periods=1).agg(['mean', 'std', 'count'])
        return roll.shift(short_window)

    stats = df.groupby(user_col)['abs_err'].apply(hist_stats).reset_index(level=0, drop=True)
    stats.columns = ['hist_mean', 'hist_std', 'hist_n']
    df = pd.concat([df, stats], axis=1)

    # global priors (use non-null historical rows)
    global_mean = df['hist_mean'].mean(skipna=True)
    global_std = df['hist_std'].mean(skipna=True)

    # shrinkage: weight = n / (n + alpha)
    weight = df['hist_n'] / (df['hist_n'] + shrink_alpha)
    df['mean_shrunk'] = df['hist_mean'].fillna(global_mean) * weight + global_mean * (1 - weight)
    df['std_shrunk'] = df['hist_std'].fillna(global_std) * weight + global_std * (1 - weight)

    # threshold and flag (also handle zero std)
    df['threshold'] = df['mean_shrunk'] + 3 * df['std_shrunk'].fillna(0)
    df['is_anomalous'] = df['rolling_mae'] > df['threshold']

    return df[[user_col, time_col, 'rolling_mae', 'mean_shrunk', 'std_shrunk', 'threshold', 'is_anomalous']]

import math, json, torch
from collections import defaultdict
import numpy as np

def tensor_stats(t):
    t = t.detach().cpu()
    a = t.view(-1).numpy()
    return {
        "mean": float(np.mean(a)),
        "std": float(np.std(a)),
        "p0": float(np.min(a)),
        "p1": float(np.percentile(a,1)),
        "p50": float(np.percentile(a,50)),
        "p99": float(np.percentile(a,99)),
        "p100": float(np.max(a)),
        "zeros_frac": float((a==0).sum()/a.size)
    }

def diagnose(model, optimizer, scheduler=None):
    report = {"layers":{}, "optimizer":{}, "alerts":[]}
    total_param_norm = 0.0
    total_grad_norm = 0.0
    zero_grad_layers = 0
    for name, p in model.named_parameters():
        stats = {}
        p_norm = p.data.norm(2).item() if p.data.numel()>0 else 0.0
        stats['param_l2'] = p_norm
        total_param_norm += p_norm**2

        if p.grad is None:
            stats['has_grad'] = False
            zero_grad_layers += 1
            stats['grad_l2'] = None
        else:
            g = p.grad
            grads_nan = torch.isnan(g).any().item()
            grads_inf = torch.isinf(g).any().item()
            g_norm = g.norm(2).item()
            stats.update({"has_grad":True, "grad_l2":g_norm,
                          "grad_has_nan":bool(grads_nan),
                          "grad_has_inf":bool(grads_inf)})
            total_grad_norm += g_norm**2

            # weight & grad distributions (sample if large)
            stats['param_stats'] = tensor_stats(p.data if p.data.numel()<=1e6 else p.data.view(-1)[:100000])
            stats['grad_stats'] = tensor_stats(g if g.numel()<=1e6 else g.view(-1)[:100000])

            # heuristics
            if grads_nan or grads_inf:
                report['alerts'].append(f"{name}: grad NaN/Inf")
            if g_norm < 1e-6:
                report['alerts'].append(f"{name}: vanishing grad (L2 {g_norm:.2e})")
            if g_norm > 1e3:
                report['alerts'].append(f"{name}: exploding grad (L2 {g_norm:.2e})")
            if stats['param_stats']['zeros_frac'] > 0.99:
                report['alerts'].append(f"{name}: >99% weights zeroed")
        report['layers'][name]=stats

    report['total_param_l2'] = math.sqrt(total_param_norm)
    report['total_grad_l2'] = math.sqrt(total_grad_norm)
    report['zero_grad_layers'] = zero_grad_layers
    # optimizer / scheduler
    lrs = set()
    for i, g in enumerate(optimizer.param_groups):
        lr = g.get("lr", None)
        lrs.add(lr)
    report['optimizer']['lrs'] = list(lrs)
    if scheduler is not None:
        # try common attributes
        report['scheduler'] = {"last_lr": getattr(scheduler, "get_last_lr", lambda: None)() if hasattr(scheduler,"get_last_lr") else None}
    # heuristics summary
    if report['total_grad_l2'] < 1e-5:
        report['alerts'].append(f"Global vanishing grads (total L2 {report['total_grad_l2']:.2e})")
    if report['total_grad_l2'] > 1e4:
        report['alerts'].append(f"Global exploding grads (total L2 {report['total_grad_l2']:.2e})")
    if len(report['optimizer']['lrs'])==1 and list(report['optimizer']['lrs'])[0]==0:
        report['alerts'].append("Learning rate is zero")
    return report

# Example usage:
# report = diagnose(model, optimizer, scheduler)
# print(json.dumps(report, indent=2))