Incident Response or Debugging Story Questions

# assumes kubernetes client configured
def health_check(pod):
    # run readiness/liveness checks + app-level probe (e.g., SQL ping, raft status)
    return pod.status.ready and app_probe_ok(pod)

def is_safe_to_restart(pod, statefulset):
    # preserve quorum: count healthy replicas before action
    healthy = count_healthy(statefulset)
    replicas = statefulset.spec.replicas
    # require at least floor(replicas/2)+1 healthy excluding this pod if possible
    required = (replicas // 2) + 1
    if healthy - (1 if health_check(pod) else 0) < required:
        return False
    # avoid restarting leader or primary without transfer
    if is_leader(pod):
        return transfer_leadership(pod)
    return True

def restart_with_backoff(pod, attempt=0):
    backoff = min(60, 2**attempt)
    sleep(backoff)
    cordon_node(pod.spec.nodeName)  # optional
    evict_or_delete_pod(pod)
    wait_for_replacement(pod, timeout=120)
    if not health_check(new_pod):
        if attempt >= 5:
            open_incident(pod, "restart_failed")
            return False
        return restart_with_backoff(pod, attempt+1)
    return True

def runbook(statefulset):
    degraded = [p for p in pods(statefulset) if not health_check(p)]
    for pod in degraded:
        if not is_safe_to_restart(pod, statefulset):
            log("unsafe to restart", pod)
            continue
        if not circuit_breaker_allows():
            log("circuit breaker tripped")
            break
        success = restart_with_backoff(pod)
        if not success:
            rollback_changes(statefulset)  # restore previous config, scale up, alert
            break