Approach: Walk the directory tree with os.walk (followlinks optional), use python-magic if available (fallback to reading header bytes and basic signatures), compare detected mime/type or extension-based expected type, stream file in chunks to compute SHA-256, and write CSV incrementally to avoid memory pressure. Use try/except to skip unreadable files and limit recursion depth if needed. Parallelism optional (thread pool) for large trees.python
import os, csv, hashlib
try:
import magic
MAGIC = magic.Magic(mime=True)
except Exception:
MAGIC = None
CHUNK = 8192
EXT_MAP = {'.jpg':'image/jpeg', '.jpeg':'image/jpeg', '.png':'image/png', '.pdf':'application/pdf', '.txt':'text/plain'}
def detect_type(path):
if MAGIC:
try:
return MAGIC.from_file(path)
except Exception:
pass
# fallback: read first bytes
with open(path,'rb') as f:
hdr = f.read(16)
if hdr.startswith(b'\xFF\xD8\xFF'): return 'image/jpeg'
if hdr.startswith(b'\x89PNG\r\n\x1a\n'): return 'image/png'
if hdr.startswith(b'%PDF-'): return 'application/pdf'
return 'application/octet-stream'
def sha256_stream(path):
h = hashlib.sha256()
with open(path,'rb') as f:
for chunk in iter(lambda: f.read(CHUNK), b''):
h.update(chunk)
return h.hexdigest()
def process(root, out_csv, followlinks=False):
with open(out_csv,'w', newline='') as csvf:
w = csv.writer(csvf)
w.writerow(['path','detected_type','extension','sha256'])
for dirpath, dirs, files in os.walk(root, followlinks=followlinks):
for name in files:
full = os.path.join(dirpath, name)
try:
if os.path.islink(full):
# optionally resolve or skip
target = os.readlink(full)
detected = detect_type(full)
ext = os.path.splitext(name)[1].lower()
expected = EXT_MAP.get(ext)
if expected and expected != detected or (not expected and detected != 'application/octet-stream'):
s = sha256_stream(full)
w.writerow([full, detected, ext, s])
except Exception:
continue
