Performance Monitoring & Observability Questions

Work per second = 500 req/s * 0.03 s = 15 core-seconds/s => 15 cores
With 50% headroom => 15 * 1.5 = 22.5 cores
With 1.5x redundancy => 22.5 * 1.5 = 33.75 cores

Servers needed = ceil(33.75 / 4) = ceil(8.4375) = 9 servers

# server
iperf3 -s
# client
iperf3 -c server -P 8 -t 60
# UDP
iperf3 -c server -u -b 1G -t 30

mtr --report --report-cycles 100 <remote>

tcpdump -i eth0 -s 0 -w hostA.pcap host hostB
tcpdump -i eth0 -s 0 -w lb.pcap host hostB

#!/usr/bin/env bash
set -euo pipefail

# Safe, idempotent disk I/O triage template
# NOTE: runs read-only commands; do NOT redirect output to files on disk.
# Run as root if available to get per-process I/O details (iotop/pidstat).

TIMESTAMP="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
echo "DISK I/O TRIAGE - $TIMESTAMP"
echo

# 1) Summary: iostat extended (device utilization, await, svctm, %util)
echo "== iostat -x (5s avg) =="
# run short sample then exit; falls back if iostat missing
if command -v iostat >/dev/null 2>&1; then
  iostat -x 1 3 | sed -n '1,200p'
else
  echo "iostat not installed"
fi
echo

# 2) Per-process I/O: use pidstat if available (reads /proc)
echo "== pidstat -d (per-process I/O) =="
if command -v pidstat >/dev/null 2>&1; then
  # sample twice to get recent rates
  pidstat -d 1 2 | sed -n '1,200p'
else
  # fallback: list top processes by read/write bytes from /proc (read-only)
  echo "pidstat not installed; using /proc parsing (may require root for full info):"
  ps -eo pid,uid,comm --sort=-%mem | head -n 10
  # attempt to read io stats per pid
  for pid in $(ps -eo pid --no-headers | head -n 20); do
    if [ -r /proc/$pid/io ]; then
      awk -v P=$pid '/read_bytes|write_bytes/ {printf "pid:%s %s %s\n", P, $1, $2}' /proc/$pid/io
    fi
  done
fi
echo

# 3) Top I/O by process using iotop if present (requires root)
echo "== iotop (snapshot) =="
if command -v iotop >/dev/null 2>&1; then
  # iotop needs root; run non-interactive snapshot for 3 seconds
  iotop -boktq 3 | sed -n '1,200p'
else
  echo "iotop not available"
fi
echo

# 4) Recent kernel messages (dmesg) for disk errors/hangs
echo "== recent dmesg lines (disk, ata, sd, nvme, error) =="
# dmesg is read-only; show last 200 lines and filter relevant keywords
if command -v dmesg >/dev/null 2>&1; then
  dmesg -T | tail -n 200 | egrep -i "ext4|xfs|sd[a-z]|nvme|ata|I/O error|error|fail|timeout" || true
else
  echo "dmesg not available"
fi
echo

# 5) Quick filesystem free space (read-only) to rule out fullness causing issues
echo "== df -h (mounted filesystems) =="
df -h --output=source,fstype,size,used,avail,pcent,target | sed -n '1,200p'
echo

# 6) Prioritized short report (human-readable) - best-effort synthesis
echo "== PRIORITIZED FINDINGS (short) =="
# Device with highest utilization from iostat (best-effort)
if command -v iostat >/dev/null 2>&1; then
  echo "- High-util devices (from iostat %util):"
  iostat -x 1 2 | awk '/^Device/ {f=1; next} f && NF {print $1, $NF}' | sort -k2 -nr | head -n 5 | sed 's/^/  - /'
else
  echo "- iostat not available to determine device utilization"
fi

# Top offenders by read/write bytes (best-effort)
echo "- Top processes by read/write bytes (if available):"
if command -v pidstat >/dev/null 2>&1; then
  pidstat -d 1 1 | awk 'NR>3 {printf "  - pid:%s cmd:%s rd/s:%s wr/s:%s\n",$2,$8,$5,$6}' | head -n 5
else
  # report from /proc parsing above
  echo "  - See pid /proc io section above"
fi

# dmesg critical warnings
echo "- Kernel errors or timeouts found (if any) above under dmesg section"
echo
echo "End of triage. Recommended next steps:"
echo "  1) If critical errors present in dmesg (I/O errors, timeouts) open hardware ticket."
echo "  2) If a single process is causing high sustained I/O, consider throttling or restarting that service during maintenance window."
echo "  3) If device %util ~100% and latency high (await), consider offloading I/O or scaling storage."
echo