Approach: walk the repository (skip .git), collect Markdown files, parse title (first H1), tags (YAML frontmatter or inline tag markers like #tag), abstract (first 30 words of content), and last_modified from git (fall back to file mtime). Handle binary/non-text files and missing git metadata gracefully. For large trees, minimize git subprocess calls by querying git once or using parallel workers.python
import os
import re
import json
import subprocess
from pathlib import Path
from datetime import datetime
MD_EXT = {'.md', '.markdown'}
INLINE_TAG_RE = re.compile(r'(?:#|@tag:)([A-Za-z0-9-_]+)') # example inline markers
def is_text_file(path, blocksize=512):
try:
with open(path, 'rb') as f:
block = f.read(blocksize)
if b'\x00' in block:
return False
return True
except Exception:
return False
def git_last_commit_date(path):
try:
# returns ISO 8601 date if available
res = subprocess.run(
['git', 'log', '-1', '--format=%cI', '--', str(path)],
capture_output=True, text=True, check=False
)
out = res.stdout.strip()
if out:
return out
except Exception:
pass
return None
def parse_frontmatter(text):
if text.startswith('---'):
parts = text.split('---', 2)
if len(parts) >= 3:
fm = parts[1]
# simple YAML tag line matching: tags: [a,b] or tags: a, b
m = re.search(r'^tags:\s*(?:\[(.*?)\]|(.+))', fm, re.MULTILINE)
if m:
raw = m.group(1) or m.group(2)
tags = [t.strip().strip('"\'') for t in re.split(r',\s*', raw) if t.strip()]
return tags, parts[2].lstrip()
return [], parts[2].lstrip()
return None, text
def extract_title_and_abstract(text):
title = None
for line in text.splitlines():
if line.strip().startswith('# '):
title = line.strip()[2:].strip()
break
words = re.findall(r'\w+|\S', text)
abstract = ' '.join(words[:30])
return title or '', abstract
def find_markdown_files(root):
for dirpath, dirnames, filenames in os.walk(root):
# skip .git
dirnames[:] = [d for d in dirnames if d != '.git']
for fn in filenames:
p = Path(dirpath) / fn
if p.suffix.lower() in MD_EXT:
yield p
def build_index(root):
index = []
for p in find_markdown_files(root):
try:
if not is_text_file(p):
continue
text = p.read_text(encoding='utf-8', errors='replace')
front_tags, body = parse_frontmatter(text)
inline_tags = INLINE_TAG_RE.findall(text)
tags = sorted(set((front_tags or []) + inline_tags))
title, abstract = extract_title_and_abstract(text if front_tags is None else body)
last_mod = git_last_commit_date(p)
if not last_mod:
# fallback to filesystem mtime
last_mod = datetime.fromtimestamp(p.stat().st_mtime).isoformat()
index.append({
'path': str(p.relative_to(root)),
'title': title,
'tags': tags,
'last_modified': last_mod,
'abstract': abstract
})
except Exception as e:
# log and continue
print(f"Warning: failed to process {p}: {e}")
return index
if __name__ == '__main__':
root = Path('.').resolve()
idx = build_index(root)
print(json.dumps(idx, indent=2))
