Fault Tolerance and System Resilience Questions

import time
import random

def is_retriable_status(status):
    # 5xx are retriable, 429 too. 4xx mostly not retriable except maybe 429 handled above.
    if status == 429 or 500 <= status < 600:
        return True
    return False

def retry_helper(func, max_retries=5, base_delay=0.5, max_delay=30, retry_on_exceptions=(Exception,), *args, **kwargs):
    """
    func: callable that performs the API request and returns (status, result) or raises.
    max_retries: max attempts (including first)
    base_delay: initial delay in seconds
    max_delay: cap for exponential backoff
    full jitter: wait = random.uniform(0, min(max_delay, base_delay * 2**attempt))
    """
    attempt = 0
    while attempt < max_retries:
        try:
            status, result = func(*args, **kwargs)  # pseudocode: call external API
            if 200 <= status < 300:
                return result
            # abort on non-retriable 4xx
            if 400 <= status < 500 and not is_retriable_status(status):
                raise RuntimeError(f"Non-retriable client error: {status}")
            # otherwise treat as retriable error and fall through to retry
            last_error = RuntimeError(f"Retriable status: {status}")
        except retry_on_exceptions as e:
            last_error = e

        attempt += 1
        if attempt >= max_retries:
            break
        # full jitter backoff
        cap = min(max_delay, base_delay * (2 ** attempt))
        wait = random.uniform(0, cap)
        time.sleep(wait)

    # exhausted retries
    raise last_error

# Usage example (pseudocode)
def call_external_api(record):
    # perform HTTP request, return (status_code, payload)
    # e.g., return (requests.post(...).status_code, response.json())
    pass

# Wrap the request in ingestion loop
for record in records_to_ingest:
    try:
        result = retry_helper(call_external_api, max_retries=4, base_delay=1, max_delay=10, args=(record,))
        process(result)
    except Exception as e:
        log.error("Failed to ingest record after retries", record=record, error=e)
        # decide: drop, dead-letter, or halt pipeline