Serve a model at scale

step1_vram.pyBYTES_PER_PARAM = {"fp32": 4, "fp16": 2, "bf16": 2, "int8": 1, "int4": 0.5}

def weight_vram_gb(params_billion, precision="fp16", overhead=1.2):
    """VRAM (GiB) for model weights at a precision, with a serving overhead factor.

    params_billion : parameter count in billions (70 for a 70B model)
    precision      : one of fp32/fp16/bf16/int8/int4 (MS1: bytes per parameter)
    overhead       : multiplier for activations + framework (~1.2 is typical)
    """
    bytes_per = BYTES_PER_PARAM[precision]
    raw_gb = params_billion * 1e9 * bytes_per / (1024 ** 3)
    return round(raw_gb * overhead, 1)

for prec in ("fp16", "int8", "int4"):
    print(f"70B @ {prec:<4}: {weight_vram_gb(70, prec)} GiB (with overhead)")
print("8B  @ fp16:", weight_vram_gb(8, "fp16"), "GiB")

step2_shard.pyfrom math import ceil

def _wv(params_billion, precision="fp16", overhead=1.2):        # from Step 1
    bpp = {"fp32": 4, "fp16": 2, "bf16": 2, "int8": 1, "int4": 0.5}[precision]
    return round(params_billion * 1e9 * bpp / (1024 ** 3) * overhead, 1)

def shard_plan(params_billion, precision, gpu_vram_gb=80, kv_reserve=0.30):
    """Smallest tensor-parallel degree G (power of two) that fits the weights and
    still leaves `kv_reserve` of each GPU free for the KV-cache. Returns a dict."""
    weights = _wv(params_billion, precision)
    usable_per_gpu = gpu_vram_gb * (1 - kv_reserve)   # leave room for KV-cache
    g = 1
    while weights / g > usable_per_gpu:
        g *= 2                                        # 1, 2, 4, 8, ...
    return {
        "weights_gb": weights,
        "tensor_parallel": g,
        "weights_per_gpu_gb": round(weights / g, 1),
        "kv_headroom_per_gpu_gb": round(gpu_vram_gb - weights / g, 1),
        "verdict": "single GPU" if g == 1 else f"tensor-parallel x{g}",
    }

for params, prec in [(8, "fp16"), (70, "fp16"), (70, "int4")]:
    plan = shard_plan(params, prec)
    print(f"{params}B @ {prec}: TP={plan['tensor_parallel']} "
          f"({plan['verdict']}), {plan['weights_per_gpu_gb']} GiB/GPU, "
          f"KV headroom {plan['kv_headroom_per_gpu_gb']} GiB")

step3_kvcache.pydef kv_bytes_per_token(layers, hidden, precision="fp16"):
    """KV-cache bytes for ONE token: 2 (K and V) x layers x hidden x bytes."""
    bytes_per = {"fp32": 4, "fp16": 2, "bf16": 2, "int8": 1, "int4": 0.5}[precision]
    return 2 * layers * hidden * bytes_per

def max_concurrency(kv_headroom_gb, layers, hidden, seq_len, precision="fp16"):
    """How many concurrent requests fit in the KV-cache headroom, each holding
    `seq_len` tokens of context."""
    per_token = kv_bytes_per_token(layers, hidden, precision)
    per_request = per_token * seq_len
    headroom_bytes = kv_headroom_gb * (1024 ** 3)
    return int(headroom_bytes // per_request)

# Llama-3-70B geometry: 80 layers, hidden 8192. KV headroom from Step 2 (TP x4): ~40.9 GiB/GPU.
per_tok = kv_bytes_per_token(80, 8192)
print("KV bytes/token :", per_tok)
for seq in (2048, 4096, 8192):
    c = max_concurrency(40.9, 80, 8192, seq)
    print(f"seq={seq:<5} -> max concurrent requests: {c}")

step4_fleet.pyfrom math import ceil

def replica_qps(max_concurrency, tokens_per_req, decode_tok_per_s, safe_util=0.7):
    """Safe requests/sec for ONE replica: run at `safe_util` of the concurrency
    ceiling so the queue stays short enough to meet a p99 SLO."""
    safe_concurrency = max_concurrency * safe_util
    # each request occupies a decode slot for tokens_per_req / decode_tok_per_s seconds
    seconds_per_req = tokens_per_req / decode_tok_per_s
    return safe_concurrency / seconds_per_req

def fleet_size(target_qps, per_replica_qps):
    return max(1, ceil(target_qps / per_replica_qps))

# From Step 3: 8 concurrent @ 2k context. A TP x4 70B replica decodes ~2500 tok/s aggregate.
r_qps = replica_qps(max_concurrency=8, tokens_per_req=400,
                    decode_tok_per_s=2500, safe_util=0.7)
print(f"per-replica safe QPS: {r_qps:.2f}")
for target in (10, 50, 120):
    n = fleet_size(target, r_qps)
    print(f"target {target:>3} QPS -> {n} replicas "
          f"(headroom {n * r_qps - target:.1f} QPS)")

step5_cost.pydef cost_per_1m_tokens(replicas, gpus_per_replica, gpu_hourly, tokens_per_sec):
    """Blended $/1M output tokens for the whole fleet at steady state."""
    fleet_cost_per_hour = replicas * gpus_per_replica * gpu_hourly
    tokens_per_hour = tokens_per_sec * 3600
    return fleet_cost_per_hour / tokens_per_hour * 1_000_000

# 3 replicas, TP x4 (4 GPUs each), $3.50/GPU-hour, ~2500 tok/s per replica.
replicas, gpus_each, hourly, tok_s = 3, 4, 3.50, 2500
fleet_tok_s = replicas * tok_s
selfhost = cost_per_1m_tokens(replicas, gpus_each, hourly, fleet_tok_s)
api_baseline = 3.00                                   # $/1M output tokens, list price

print(f"fleet throughput   : {fleet_tok_s} tok/s")
print(f"self-host $/1M tok  : ${selfhost:.2f}")
print(f"API baseline $/1M   : ${api_baseline:.2f}")
cheaper = "self-host" if selfhost < api_baseline else "API"
print(f"cheaper at this load: {cheaper} "
      f"({abs(selfhost - api_baseline) / api_baseline * 100:.0f}% delta)")

step6_plan.pyfrom math import ceil

BPP = {"fp32": 4, "fp16": 2, "bf16": 2, "int8": 1, "int4": 0.5}

def weight_vram_gb(params_b, prec, overhead=1.2):                       # Step 1
    return round(params_b * 1e9 * BPP[prec] / (1024 ** 3) * overhead, 1)

def shard_plan(params_b, prec, gpu_vram=80, kv_reserve=0.30):          # Step 2
    w = weight_vram_gb(params_b, prec); usable = gpu_vram * (1 - kv_reserve); g = 1
    while w / g > usable:
        g *= 2
    return g, round(w / g, 1), round(gpu_vram - w / g, 1)

def kv_per_token(layers, hidden, prec):                                # Step 3
    return 2 * layers * hidden * BPP[prec]

def max_concurrency(kv_headroom_gb, layers, hidden, seq_len, prec):    # Step 3
    return int(kv_headroom_gb * (1024 ** 3) // (kv_per_token(layers, hidden, prec) * seq_len))

def replica_qps(conc, tokens_per_req, decode_tok_s, safe_util=0.7):    # Step 4
    return (conc * safe_util) / (tokens_per_req / decode_tok_s)

def cost_per_1m(replicas, gpus_each, gpu_hourly, tok_s):               # Step 5
    return replicas * gpus_each * gpu_hourly / (tok_s * 3600) * 1_000_000


def plan_deployment(name, params_b, layers, hidden, prec,
                    target_qps, seq_len, tokens_per_req,
                    decode_tok_s, gpu_hourly, api_baseline, gpu_vram=80):
    """Full end-to-end plan: VRAM -> shard -> concurrency -> fleet -> cost."""
    g, w_per_gpu, kv_headroom = shard_plan(params_b, prec, gpu_vram)     # 1 + 2
    conc = max_concurrency(kv_headroom, layers, hidden, seq_len, prec)   # 3
    r_qps = replica_qps(conc, tokens_per_req, decode_tok_s)              # 4
    replicas = max(1, ceil(target_qps / r_qps))                         # 4
    fleet_tok_s = replicas * decode_tok_s                               # 5
    selfhost = cost_per_1m(replicas, g, gpu_hourly, fleet_tok_s)        # 5
    winner = "self-host" if selfhost < api_baseline else "API"

    print(f"=== deployment plan: {name} @ {prec} ===")
    print(f"  weights VRAM      : {weight_vram_gb(params_b, prec)} GiB")
    print(f"  sharding          : TP x{g} ({w_per_gpu} GiB/GPU, {kv_headroom} GiB KV headroom)")
    print(f"  max concurrency   : {conc} req/replica @ {seq_len} ctx")
    print(f"  per-replica QPS   : {r_qps:.2f}")
    print(f"  fleet             : {replicas} replicas x {g} GPU = {replicas * g} GPUs")
    print(f"  fleet throughput  : {fleet_tok_s} tok/s (target {target_qps} QPS)")
    print(f"  self-host $/1M    : ${selfhost:.2f}   API baseline ${api_baseline:.2f}")
    print(f"  recommendation    : {winner} at this load")
    return {"tp": g, "replicas": replicas, "gpus": replicas * g, "usd_per_1m": round(selfhost, 2)}


plan_deployment(
    name="Llama-3-70B", params_b=70, layers=80, hidden=8192, prec="fp16",
    target_qps=50, seq_len=4096, tokens_per_req=400,
    decode_tok_s=2500, gpu_hourly=3.50, api_baseline=3.00)

serve_tp2.sh# ▶ needs GPUs: 2x H100 (80 GiB), pip install vllm
# Serve Llama-3-70B tensor-parallel across 2 GPUs, matching the plan above.
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3-70B-Instruct \
  --tensor-parallel-size 4 \        # TP degree from Step 2
  --max-model-len 4096 \            # context length from Step 3
  --max-num-seqs 4 \                # concurrency ceiling from Step 3
  --gpu-memory-utilization 0.90 \   # leave headroom for the KV-cache
  --enable-prefix-caching            # reuse shared prefixes' KV (MS3/MS4)
# Then run a load test at rising QPS and confirm p99 holds under the SLO
# you sized the fleet for in Step 4 (this is one replica; run N of them).