Approach (brief): define a binary snapshot format with a fixed magic, version (major.minor), header JSON for metadata (timestamp, node-count), payload (compact preorder with null markers), and a SHA-256 checksum of header+payload appended. Reader validates magic, checksum, then compares versions: if same -> accept; if older -> run an upgrade path to current version; if newer and incompatible -> reject.Python implementation:python
import json, struct, hashlib, time
from typing import Optional, List, Tuple
MAGIC = b'BSTSNAP\x00' # 8 bytes
HEADER_FMT = '!I' # 4-byte header length
class Node:
def __init__(self, v:int, left=None, right=None):
self.v = v; self.left = left; self.right = right
def _serialize_tree_preorder(root:Optional[Node]) -> List[int]:
# Use sentinel None as marker
out=[]
def dfs(n):
if n is None:
out.append(None); return
out.append(n.v)
dfs(n.left); dfs(n.right)
dfs(root); return out
def write_snapshot(root:Optional[Node], version: Tuple[int,int], path:str):
payload = json.dumps(_serialize_tree_preorder(root), separators=(',',':')).encode()
header = {
'version': f'{version[0]}.{version[1]}',
'ts': int(time.time()),
'nodes': payload.count(b',')+1 if payload else 0
}
header_bytes = json.dumps(header, separators=(',',':')).encode()
hdr_len = struct.pack(HEADER_FMT, len(header_bytes))
core = MAGIC + hdr_len + header_bytes + payload
checksum = hashlib.sha256(core).digest()
with open(path,'wb') as f:
f.write(core + checksum)
def _parse_version(s:str) -> Tuple[int,int]:
a,b = s.split('.'); return int(a), int(b)
def upgrade_payload(old_payload:bytes, old_version:Tuple[int,int], cur_version:Tuple[int,int]) -> bytes:
# Example upgrade: no-op or transformation code per version
# For demo, assume backward-compatible changes handled here
return old_payload
def read_snapshot(path:str, cur_version:Tuple[int,int]) -> Optional[Node]:
with open(path,'rb') as f:
data = f.read()
if len(data) < len(MAGIC)+4+32: raise ValueError('corrupt/too small')
if not data.startswith(MAGIC): raise ValueError('bad magic')
idx = len(MAGIC)
hdr_len = struct.unpack(HEADER_FMT, data[idx:idx+4])[0]; idx+=4
header_bytes = data[idx:idx+hdr_len]; idx+=hdr_len
# remaining: payload + checksum
checksum = data[-32:]; payload = data[idx:-32]
core = data[:len(data)-32]
if hashlib.sha256(core).digest() != checksum: raise ValueError('checksum mismatch')
header = json.loads(header_bytes.decode())
file_version = _parse_version(header['version'])
if file_version == cur_version:
pass
elif file_version < cur_version:
payload = upgrade_payload(payload, file_version, cur_version)
else:
# newer file: if minor compatible (same major) accept, else reject
if file_version[0] > cur_version[0]:
raise ValueError('snapshot from newer incompatible major version')
# rebuild tree
arr = json.loads(payload.decode())
it = iter(arr)
def build():
try:
v = next(it)
except StopIteration:
return None
if v is None: return None
node = Node(v)
node.left = build(); node.right = build()
return node
return build()
Key concepts:- Magic bytes + header length let reader locate header reliably.- Header holds version, timestamp, node count (helps quick sanity checks).- SHA-256 of (magic+hdr_len+header+payload) ensures integrity.- Semantic versioning: major changes incompatible; minor/patch should be backward-compatible.- Upgrade hooks implement deterministic migrations (payload transforms).Complexity:- Time: O(n) to serialize/deserialize (n = nodes).- Space: O(n) for payload; header constant.Performance trade-offs:- JSON payload is simple/readable but larger; binary packing (struct, compact varint) reduces size/CPU for parsing.- Checksum cost: SHA-256 is secure but CPU heavier; CRC32 cheaper but weaker. For large snapshots, compute checksum streaming (hashlib update) to avoid duplicating memory.- Compression reduces I/O but increases CPU and complicates partial reads/stream validation (compute checksum on compressed bytes vs original — pick one and document).- Upgrade on read can be costly; prefer lazy upgrade or online migration strategies.SRE CI validation strategy:- Add snapshot unit tests that create, write, read and validate checksums and round-trip equality.- Use a corpus of golden snapshots across versions; CI runs compatibility matrix: for each older version snapshot, assert reader accepts/upgrades.- Fuzz snapshots (bit flips) to validate checksum rejection and monitor failure modes.- Measure checksum failure rate in production; alert on increases.- Automate streaming checksum verification in backup jobs; include snapshot metadata (size, node-count, checksum) in monitoring dashboards.- For large systems, test performance under realistic sizes in CI (benchmark), and ensure snapshotting doesn't breach SLOs (I/O, CPU).