Approach: implement a tail -F style watcher that opens the file, seeks to end, reads new lines, and monitors file identity (inode + device) and size to detect rotation or truncation. On rotation (inode change) reopen the path and continue; on truncation (size smaller) seek to start of file.python
import os
import time
from typing import Callable
def follow(path: str, callback: Callable[[str], None], poll_interval=0.2):
"""
Follow a log file, calling callback for each new line.
Handles rotation (inode change), truncation, and rapid rotations.
"""
def stat(fpath):
try:
s = os.stat(fpath)
return (s.st_ino, s.st_dev, s.st_size)
except FileNotFoundError:
return None
fh = None
last_id = None
while True:
cur_stat = stat(path)
if cur_stat is None:
# file missing (maybe between rotations) — wait and retry
if fh:
fh.close()
fh = None
last_id = None
time.sleep(poll_interval)
continue
if fh is None:
# open new file and, if it is the same inode as last seen, keep position; else start at 0
fh = open(path, 'r', encoding='utf-8', errors='replace')
ino = (cur_stat[0], cur_stat[1])
if ino == last_id:
# same file reopened (rare), seek to previous end
fh.seek(cur_stat[2])
else:
# new file: start at end (like tail -F); if you want to read historical lines change to 0
fh.seek(0, os.SEEK_END)
last_id = ino
# read lines
line = fh.readline()
if line:
callback(line.rstrip('\n'))
continue
# no data: check for rotation or truncation
latest = stat(path)
if latest is None:
# file disappeared
fh.close()
fh = None
last_id = None
time.sleep(poll_interval)
continue
latest_id = (latest[0], latest[1])
if latest_id != last_id:
# inode changed -> rotation. Keep reading remainder of old file first (if any), then switch.
# close old handle and open new file
fh.close()
# If rotated file still exists under old name? Usually it's renamed; reopen path will be new file
fh = open(path, 'r', encoding='utf-8', errors='replace')
last_id = latest_id
# start at beginning of new file
fh.seek(0, os.SEEK_END)
continue
# same inode but size decreased -> truncation
if latest[2] < fh.tell():
fh.seek(0)
continue
time.sleep(poll_interval)
Key points:- Use inode+device to detect rotation reliably; file path can be replaced while old file still continues being written to.- Truncation (logtruncate) detected by file size < current position — seek to start.- Rapid rotations: we always compare stat each loop and reopen as soon as inode differs; small poll_interval reduces missed lines but increases CPU.- Keep file handle reading until EOF before switching to avoid losing final lines when rotated.Complexity: O(1) memory; CPU proportional to polling frequency. Edge cases: permission errors, very high write rates (use smaller interval or event-based APIs like inotify on Linux), multi-line writes (use buffering), encoding errors handled with errors='replace'.