To estimate cost-per-request and recommended instance count, we first compute each instance's effective inference throughput (in requests/sec) given its raw FLOPS, the model FLOPS per inference, and an operational GPU utilization cap. Then we size the number of instances to meet target_requests_per_second, compute hourly compute cost for that fleet, and add egress cost per request.python
import math
def estimate_cost_per_request(
model_flops_per_inference,
instance_flops, # FLOPS per second of one instance (e.g., 1e14)
instance_hourly_price, # $ per hour for one instance
expected_gpu_utilization, # expected utilization fraction (0-1)
network_bytes_per_request, # bytes sent per inference (egress)
egress_price_per_gb, # $ per GB egress
target_requests_per_second,
utilization_cap=0.7 # cap to avoid driving GPUs to 100%
):
"""
Returns (dollars_per_request, recommended_instance_count)
Assumptions & calculations:
- effective_util = min(expected_gpu_utilization, utilization_cap)
- flops_per_second_available = instance_flops * effective_util
- throughput_per_instance_rps = flops_per_second_available / model_flops_per_inference
(if model_flops_per_inference==0 -> error)
- instances_needed = ceil(target_rps / throughput_per_instance_rps)
- compute_cost_per_request = (instances_needed * instance_hourly_price) / (target_rps * 3600)
(hourly cost spread across requests per hour = target_rps * 3600)
- egress_cost_per_request = (network_bytes_per_request / (1024**3)) * egress_price_per_gb
- total = compute + egress
"""
# Basic validations
if model_flops_per_inference <= 0:
raise ValueError("model_flops_per_inference must be > 0")
if instance_flops <= 0:
raise ValueError("instance_flops must be > 0")
if target_requests_per_second <= 0:
raise ValueError("target_requests_per_second must be > 0")
if not (0 <= expected_gpu_utilization <= 1):
raise ValueError("expected_gpu_utilization must be in [0,1]")
if not (0 < utilization_cap <= 1):
raise ValueError("utilization_cap must be in (0,1]")
effective_util = min(expected_gpu_utilization, utilization_cap)
flops_available_per_sec = instance_flops * effective_util
throughput_per_instance = flops_available_per_sec / model_flops_per_inference
if throughput_per_instance <= 0:
raise ValueError("Throughput per instance is zero; check FLOPS inputs")
instances_needed = max(1, math.ceil(target_requests_per_second / throughput_per_instance))
# Compute cost per request:
requests_per_hour = target_requests_per_second * 3600.0
hourly_compute_cost = instances_needed * instance_hourly_price
compute_cost_per_request = hourly_compute_cost / requests_per_hour
# Egress cost per request:
egress_gb_per_request = network_bytes_per_request / (1024**3)
egress_cost_per_request = egress_gb_per_request * egress_price_per_gb
total_cost_per_request = compute_cost_per_request + egress_cost_per_request
return total_cost_per_request, instances_needed
# Example usage:
# cost, instances = estimate_cost_per_request(
# model_flops_per_inference=5e12,
# instance_flops=1.2e14,
# instance_hourly_price=3.5,
# expected_gpu_utilization=0.6,
# network_bytes_per_request=1_000_000, # 1 MB
# egress_price_per_gb=0.09,
# target_requests_per_second=50,
# utilization_cap=0.7
# )
# print(cost, instances)
