Operational Excellence and Resilience Questions

- alert: PaymentsHigh5xxErrorRate
  expr: |
    (
      sum by (instance) (
        increase(http_requests_total{service="payments",status=~"5.."}[5m])
      )
      /
      sum by (instance) (
        increase(http_requests_total{service="payments"}[5m])
      )
    ) > 0.01
  for: 3m
  labels:
    severity: page
    team: payments
  annotations:
    summary: "Payments service 5xx rate >1% over 5m on {{ $labels.instance }}"
    description: "5xx error rate is {{ $value }} (>1%) for 3m. Investigate upstreams, deployments, or load spikes."

import time
import random

def retry_request(func, max_retries=5, base_delay=0.5, max_delay=30, seed=None):
    """
    Calls func() and retries on exception using exponential backoff with full jitter.
    - func: callable with zero args that performs the HTTP request (may raise).
    - max_retries: maximum retry attempts (total tries = max_retries + 1).
    - base_delay: initial backoff base in seconds.
    - max_delay: maximum delay cap in seconds.
    - seed: optional seed for deterministic jitter (useful in tests).
    """
    rng = random.Random(seed)
    attempt = 0
    last_exc = None

    while attempt <= max_retries:
        try:
            return func()
        except Exception as e:
            last_exc = e
            if attempt == max_retries:
                break
            # exponential cap then full jitter
            cap = min(max_delay, base_delay * (2 ** attempt))
            delay = rng.uniform(0, cap)
            time.sleep(delay)
            attempt += 1

    # Exhausted retries; re-raise the last exception
    raise last_exc