Situation/Problem:On-call engineers repeatedly ran a manual bi-hourly check to reconcile Prometheus alerts vs. PagerDuty incident states and silence noisy alerts during deploy windows — ~10 minutes per run and occasional missed silences led to false pages.Approach:I wrote a Python CLI that queries Prometheus and PagerDuty, reconciles active alerts with open incidents, creates/clears silences in Alertmanager for deploy windows, and posts status to Slack. It runs in cron and is callable from our CI and a chatops slash command.Code (key parts):python
#!/usr/bin/env python3
import requests, os, logging, time
PD_TOKEN = os.environ['PD_TOKEN']
AM_URL = os.environ['ALERTMANAGER']
PROM_URL = os.environ['PROM']
SLACK_WEBHOOK = os.environ['SLACK']
def get_prom_alerts():
r = requests.get(f"{PROM_URL}/api/v1/alerts", timeout=10)
return r.json()['data']
def get_pd_incidents():
r = requests.get("https://api.pagerduty.com/incidents",
headers={'Authorization':f"Token token={PD_TOKEN}", 'Accept':'application/vnd.pagerduty+json;version=2'},
params={'statuses[]':'triggered'}, timeout=10)
return r.json()['incidents']
def create_silence(matchers, duration=3600):
payload = {"matchers": matchers, "startsAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"endsAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time()+duration)),
"createdBy":"automation","comment":"auto-deploy window"}
return requests.post(f"{AM_URL}/api/v2/silences", json=payload, timeout=10)
# main: reconcile and silence
