Approach (brief): stream the file line-by-line, parse timestamp to hour bucket, normalize the error message into a signature (regex substitutions to remove variable parts), then increment a per-hour counter for that signature. Keep memory bounded by periodically flushing older hours to disk or using an on-disk/key-value store.Assumptions:- Memory: limited to fit a few hours of aggregated counters in memory; raw file too large to hold.- Input: one syslog line per record, timestamp at start in ISO or common syslog format.- Timestamp parsing: use dateutil or strptime with fallback formats.- Normalization: numbers, hex, UUIDs, file paths, timestamps within message are variable parts.Python script (compact):python
import re
from collections import defaultdict, Counter
from datetime import datetime
from dateutil import parser
# regexes to normalize variable parts
RE_NUM = re.compile(r'\b\d+\b')
RE_HEX = re.compile(r'\b0x[0-9a-fA-F]+\b')
RE_UUID = re.compile(r'\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b')
RE_PATH = re.compile(r'(/[A-Za-z0-9._-]+)+')
def normalize(msg):
msg = RE_UUID.sub('<UUID>', msg)
msg = RE_HEX.sub('<HEX>', msg)
msg = RE_PATH.sub('<PATH>', msg)
msg = RE_NUM.sub('<N>', msg)
return ' '.join(msg.split()).lower()
def hour_bucket(ts):
return ts.strftime('%Y-%m-%dT%H:00:00')
def stream_signatures(path, flush_hours=4):
counts = defaultdict(Counter) # hour -> Counter(signature->count)
with open(path, 'r', encoding='utf-8', errors='replace') as f:
for line in f:
try:
# assume timestamp at beginning; fallback to parser
ts_str, _, rest = line.partition(' ')
ts = parser.parse(ts_str)
except Exception:
try:
ts = parser.parse(line) # try anywhere
rest = line
except Exception:
continue
hour = hour_bucket(ts)
sig = normalize(rest)
counts[hour][sig] += 1
# flush logic: drop/serialize counts older than flush_hours
if len(counts) > flush_hours:
oldest = sorted(counts.keys())[0]
# persist to disk / DB - here we print then remove
for s, c in counts[oldest].items():
print(oldest, c, s)
del counts[oldest]
# final flush
for h, ctr in counts.items():
for s, c in ctr.items():
print(h, c, s)
# Example usage:
# stream_signatures('/var/log/syslog')
