Cloud Monitoring and Troubleshooting Questions

# python-like pseudocode
import os, time, requests, json
import boto3
from botocore.exceptions import ClientError

# Config
LOG_GROUP = "/aws/service/my-service"
QUERY_STRING = 'fields @timestamp, @message | filter level="ERROR" | stats count() as error_count'
TIME_WINDOW_MIN = 5
THRESHOLD = 100
PAGERDUTY_ROUTING_KEY = os.environ['PD_ROUTING_KEY']  # Events V2 integration key
AWS_REGION = "us-east-1"

logs = boto3.client('logs', region_name=AWS_REGION)  # uses IAM role or env creds

def run_insights_query():
    now = int(time.time())
    start = now - TIME_WINDOW_MIN*60
    try:
        resp = logs.start_query(
            logGroupName=LOG_GROUP,
            startTime=start,
            endTime=now,
            queryString=QUERY_STRING,
            limit=1000
        )
        query_id = resp['queryId']
        # poll until complete
        for _ in range(30):
            r = logs.get_query_results(queryId=query_id)
            status = r['status']
            if status in ('Complete','Failed','Cancelled'):
                return r
            time.sleep(1)
        raise TimeoutError("Insights query timed out")
    except ClientError as e:
        raise

def extract_count(results):
    # results: list of rows with field/value pairs
    if results.get('status') != 'Complete': return 0
    rows = results.get('results', [])
    if not rows: return 0
    # assume first row has error_count field
    for field in rows[0]:
        if field['field'] == 'error_count':
            return int(field['value'])
    # fallback: sum counts if multiple groups
    total = 0
    for row in rows:
        for f in row:
            if f['field']=='error_count':
                total += int(f['value'])
    return total

def create_pagerduty_incident(summary, details):
    url = "https://events.pagerduty.com/v2/enqueue"
    payload = {
        "routing_key": PAGERDUTY_ROUTING_KEY,
        "event_action": "trigger",
        "payload": {
            "summary": summary,
            "severity": "critical",
            "source": "cloudwatch-insights",
            "custom_details": details
        }
    }
    headers = {"Content-Type":"application/json"}
    r = requests.post(url, headers=headers, data=json.dumps(payload), timeout=10)
    r.raise_for_status()
    return r.json()

def handler():
    results = run_insights_query()
    count = extract_count(results)
    if count >= THRESHOLD:
        summary = f"Spike detected: {count} errors in last {TIME_WINDOW_MIN}m for my-service"
        details = {
            "log_group": LOG_GROUP,
            "window_min": TIME_WINDOW_MIN,
            "error_count": count,
            "query": QUERY_STRING
        }
        resp = create_pagerduty_incident(summary, details)
        return resp
    return {"status":"ok","error_count":count}

# Entry point for scheduler (lambda/cron)
if __name__ == "__main__":
    print(handler())

fields @timestamp, service, status
| filter status = 500 and @timestamp >= ago(24h)
| stats count() as error_count by service
| sort error_count desc
| limit 10

timestamp >= "2025-12-05T00:00:00Z"  // replace with now()-24h in UI quick filters
AND jsonPayload.status = 500
| jsonPayload.service as service
| count_by(service) // Use UI "Group by" service and Aggregation=count
| sort_desc(count)
| limit 10