Fault Tolerance and Failure Scenarios Questions

import time
import threading
import random
from typing import Callable, Any

_thread_local = threading.local()

def _get_thread_random():
    if not hasattr(_thread_local, "rand"):
        # seed from system randomness
        _thread_local.rand = random.Random()
    return _thread_local.rand

def retry_with_full_jitter(callable_fn: Callable[[], Any],
                           max_attempts: int = 5,
                           initial_delay_ms: int = 100,
                           multiplier: float = 2.0,
                           max_delay_ms: int = 10000,
                           max_total_time_ms: int = None) -> Any:
    """
    Retry callable_fn using exponential backoff with full jitter.
    Raises last exception if all retries fail.
    """
    if max_attempts <= 0:
        raise ValueError("max_attempts must be >= 1")

    rand = _get_thread_random()
    attempt = 0
    delay_ms = float(initial_delay_ms)
    start = time.monotonic()

    last_exc = None

    while attempt < max_attempts:
        attempt += 1
        try:
            return callable_fn()
        except Exception as exc:
            last_exc = exc
            # If we've exhausted attempts, break and raise
            if attempt >= max_attempts:
                break

            # Check total elapsed time if provided
            if max_total_time_ms is not None:
                elapsed_ms = (time.monotonic() - start) * 1000.0
                if elapsed_ms >= max_total_time_ms:
                    break

            # compute capped delay and full jitter
            capped = min(delay_ms, float(max_delay_ms))
            sleep_ms = rand.uniform(0.0, capped)
            # But ensure sleeping won't exceed max_total_time_ms if provided
            if max_total_time_ms is not None:
                elapsed_ms = (time.monotonic() - start) * 1000.0
                remaining_ms = max_total_time_ms - elapsed_ms
                if remaining_ms <= 0:
                    break
                sleep_ms = min(sleep_ms, remaining_ms)

            # sleep using monotonic-aware sleep
            time.sleep(sleep_ms / 1000.0)

            # prepare next delay
            delay_ms = delay_ms * multiplier

    # If we get here, all retries failed or time exhausted
    raise last_exc

# priorities: list of key lists (critical first)
# rate_limiter.allow() enforces origin QPS
# fetch_from_origin(key) returns (value, version)
# cache.upsert(key, value, version)
for priority_bucket in priorities:
    for key in priority_bucket:
        if cache.missing_or_stale(key):
            if rate_limiter.allow():
                val, ver = fetch_from_origin(key)
                # idempotent upsert: only apply if ver > cached_ver
                cache.compare_and_set(key, val, ver)
            else:
                schedule_retry_with_jitter(key, next_slot)
    # wait small window to let system settle before next bucket

# Background reconciler continuously scans low-priority keys at low rate