Approach (brief):- Query systemd for failed units via systemctl (subprocess).- For each failed unit, check a persistent store of last-restart timestamps (SQLite) to enforce 1 restart / 10 minutes per unit.- If allowed, attempt restart via systemctl and log outcome. Store restart timestamp only on attempted restart (successful or not) to avoid tight retry loops.- Idempotent: repeated runs won't restart units within 10 minutes and will only act on units in failed state.Core algorithm: list failed units -> for each unit: if now - last_restart >= 600s -> restart -> update DB -> log.Runnable core loop (Python 3):python
#!/usr/bin/env python3
import subprocess, logging, sqlite3, time, shlex
DB = "/var/lib/unit-restart.db" # ensure writable by runner or use /tmp for testing
RATE_LIMIT_SECONDS = 600
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
def run_cmd(cmd):
return subprocess.run(shlex.split(cmd), capture_output=True, text=True)
def list_failed_units():
# shows lines like: UNIT LOAD ACTIVE SUB DESCRIPTION
r = run_cmd("systemctl --no-pager --failed --no-legend --plain")
if r.returncode != 0:
logging.error("systemctl failed: %s", r.stderr.strip())
return []
units = []
for line in r.stdout.splitlines():
parts = line.split()
if parts:
units.append(parts[0]) # first token is unit name
return units
def init_db(path=DB):
conn = sqlite3.connect(path, timeout=5)
conn.execute("CREATE TABLE IF NOT EXISTS restarts(unit TEXT PRIMARY KEY, last_ts INTEGER)")
conn.commit()
return conn
def allowed_to_restart(conn, unit):
cur = conn.execute("SELECT last_ts FROM restarts WHERE unit = ?", (unit,)).fetchone()
if not cur:
return True
return (time.time() - cur[0]) >= RATE_LIMIT_SECONDS
def record_restart(conn, unit):
ts = int(time.time())
conn.execute("INSERT INTO restarts(unit,last_ts) VALUES(?,?) ON CONFLICT(unit) DO UPDATE SET last_ts=excluded.last_ts", (unit, ts))
conn.commit()
def restart_unit(unit):
logging.info("Attempting restart of %s", unit)
r = run_cmd(f"systemctl restart {shlex.quote(unit)}")
if r.returncode == 0:
logging.info("Restart succeeded for %s", unit)
else:
logging.warning("Restart failed for %s: %s", unit, r.stderr.strip())
return r.returncode == 0
def main_once():
conn = init_db()
units = list_failed_units()
if not units:
logging.info("No failed units found.")
return
for u in units:
if allowed_to_restart(conn, u):
# record restart attempt regardless to avoid tight loops
record_restart(conn, u)
restart_unit(u)
else:
logging.info("Skipping %s due to rate limit", u)
if __name__ == "__main__":
main_once()
