SRE Fundamentals and Role Understanding Questions

# successful requests in last 5m
sum(increase(http_requests_total{service="images", code="200"}[5m])) 
/
# total requests in last 5m
sum(increase(http_requests_total{service="images"}[5m]))

- alert: ImagesServiceAvailabilityLow
  expr: |
    (
      sum(increase(http_requests_total{service="images", code="200"}[5m]))
      /
      sum(increase(http_requests_total{service="images"}[5m]))
    ) < 0.9995
    and
    sum(increase(http_requests_total{service="images"}[5m])) > 0
  for: 15m
  labels:
    severity: page
  annotations:
    summary: "Images service availability < 99.95% (5m window) for 15m"
    description: "Availability = successful/total over 5m is below 99.95% for 15 minutes."

# pseudo-python
from time import time
from collections import defaultdict, deque

# Parameters
SUPPRESS_TTL = 300        # seconds to suppress identical alerts
FREQ_WINDOW = 600         # sliding window size in seconds for frequency count
ESCALATION_RATE = 5       # factor increase threshold
ESCALATION_MIN_COUNT = 3  # minimum baseline count to consider escalation

store = {}                # fingerprint -> {last_seen, queue_of_timestamps}

def fingerprint(alert):
    key = (alert.service, normalize(alert.error), normalize(alert.stack_trace))
    return sha1_hash(key)

def on_alert(alert):
    now = time()
    fp = fingerprint(alert)
    entry = store.get(fp)
    if not entry:
        entry = {'last_seen':0, 'timestamps': deque()}
        store[fp] = entry

    # purge old timestamps from sliding window
    while entry['timestamps'] and entry['timestamps'][0] < now - FREQ_WINDOW:
        entry['timestamps'].popleft()

    # suppression: if last_seen within SUPPRESS_TTL, drop/suppress
    if now - entry['last_seen'] < SUPPRESS_TTL:
        # optionally increment counter for frequency
        entry['timestamps'].append(now)
        return "suppressed"

    # record occurrence
    entry['last_seen'] = now
    entry['timestamps'].append(now)

    # compute rate ratio against baseline (e.g., average prior window)
    baseline = estimate_baseline(fp)  # e.g., historical avg per FREQ_WINDOW
    recent = len(entry['timestamps'])
    if baseline >= ESCALATION_MIN_COUNT and recent >= baseline * ESCALATION_RATE:
        escalate(alert, fp, recent, baseline)
    else:
        notify(alert, fp)
    return "accepted"