Approach: build a Cache abstraction with a thread-safe process-local LRU-like dict plus optional external backend implementing the same interface (example: Redis). Local cache uses threading.Lock for concurrency; since WSGI typically forks multiple processes, each process has its own local cache. External backend provides cross-process sharing. TTL is enforced on both layers; local cache has a background cleaner thread.python
import threading
import time
import pickle
from typing import Any, Optional
class Backend:
def get(self, key: str) -> Optional[bytes]:
raise NotImplementedError
def set(self, key: str, value: bytes, ttl: Optional[int]) -> None:
raise NotImplementedError
class RedisBackend(Backend):
def __init__(self, redis_client):
self.r = redis_client
def get(self, key):
return self.r.get(key)
def set(self, key, value, ttl):
if ttl:
self.r.setex(key, ttl, value)
else:
self.r.set(key, value)
class ProcessLocalCache:
def __init__(self, max_items=1000, clean_interval=60):
self.store = {} # key -> (expiry_ts or None, bytes)
self.lock = threading.Lock()
self.max_items = max_items
self.cleaner = threading.Thread(target=self._cleaner, daemon=True)
self.clean_interval = clean_interval
self.cleaner.start()
def _cleaner(self):
while True:
time.sleep(self.clean_interval)
now = time.time()
with self.lock:
keys = [k for k,(exp,_) in self.store.items() if exp is not None and exp <= now]
for k in keys: del self.store[k]
def get(self, key):
now = time.time()
with self.lock:
item = self.store.get(key)
if not item: return None
exp, data = item
if exp is not None and exp <= now:
del self.store[key]
return None
return data
def set(self, key, value_bytes, ttl=None):
exp = time.time()+ttl if ttl else None
with self.lock:
if len(self.store) >= self.max_items:
# simple eviction: remove oldest expiry or arbitrary
self.store.pop(next(iter(self.store)))
self.store[key] = (exp, value_bytes)
class InferenceCache:
def __init__(self, backend: Optional[Backend]=None, local_max=1000):
self.local = ProcessLocalCache(max_items=local_max)
self.backend = backend
def get(self, key: str) -> Optional[Any]:
data = self.local.get(key)
if data is not None:
return pickle.loads(data)
if self.backend:
b = self.backend.get(key)
if b:
obj = pickle.loads(b)
# populate local for faster subsequent reads
self.local.set(key, b, ttl=None) # TTL unknown; could encode expiry in payload
return obj
return None
def set(self, key: str, value: Any, ttl: Optional[int]=None):
b = pickle.dumps(value)
self.local.set(key, b, ttl)
if self.backend:
self.backend.set(key, b, ttl)
Key points:- Thread-safety via locks for local cache; processes are isolated so no cross-process locks needed.- External backend (Redis) is pluggable for cross-process sharing.- TTL enforced locally; for accurate TTL propagation, encode expiry in payload or rely on backend TTL (Redis handles it).- Eviction here is simple—swap for LRU if needed (collections.OrderedDict or cachetools).Complexity: get/set local O(1); external depends on backend (Redis: network).Edge cases: large pickled objects, serialization errors, clock skew across machines, TTL consistency between local and external. Alternatives: use shared memory (multiprocessing.Manager), or use Redis-only with local negative-cache to avoid stale data.