{"model": "reduced-order KV-cache / attention-memory model -- HBM footprint, capacity fit, decode gating (educational)", "inputs": {"model_params_b": 8.0, "num_layers": 32, "num_kv_heads": 8, "head_dim": 128, "context_length": 8192, "batch": 32, "kv_precision_bits": 16.0, "weight_precision_bits": 16.0, "hbm_capacity_gb": 80.0, "hbm_bw_gbs": 3350.0}, "outputs": {"kv_bytes_per_token": 131072.0, "kv_kb_per_token": 128.0, "kv_per_sequence_gb": 1.0737, "kv_total_gb": 34.36, "weight_gb": 16.0, "footprint_gb": 50.36, "capacity_gb": 80.0, "utilization_percent": 62.9, "kv_fraction_percent": 68.2, "free_for_kv_gb": 64.0, "fits": true, "max_batch_at_context": 59, "max_context_at_batch": 15258, "bytes_per_step_gb": 50.36, "time_per_step_ms": 15.033, "decode_tokens_per_s": 2128.7, "bottleneck": "KV-cache-bound", "verdict": "KV-cache-dominated -- the cache outweighs the weights and is re-read every token, so decode throughput is set by cache size: long-context serving is a bandwidth problem, quantize KV or reduce concurrency to recover it"}, "profile": {"footprint_vs_ctx": [[128.0, 16.537], [155.0, 16.65], [188.0, 16.788], [228.0, 16.954], [276.0, 17.156], [334.0, 17.4], [404.0, 17.696], [490.0, 18.055], [593.0, 18.489], [719.0, 19.015], [871.0, 19.652], [1055.0, 20.424], [1278.0, 21.358], [1548.0, 22.491], [1875.0, 23.862], [2271.0, 25.524], [2751.0, 27.537], [3332.0, 29.975], [4036.0, 32.928], [4889.0, 36.506], [5922.0, 40.839], [7174.0, 46.089], [8690.0, 52.448], [10526.0, 60.15], [12751.0, 69.481], [15445.0, 80.783], [18710.0, 94.474], [22664.0, 111.058], [27453.0, 131.147], [33255.0, 155.481], [40283.0, 184.958], [48796.0, 220.664], [59108.0, 263.917], [71599.0, 316.31], [86731.0, 379.775], [105060.0, 456.653], [127262.0, 549.777], [154157.0, 662.582], [186736.0, 799.226], [226199.0, 964.748], [274003.0, 1165.25], [331908.0, 1408.125], [402052.0, 1702.327], [487018.0, 2058.704], [589942.0, 2490.395], [714616.0, 3013.316], [865638.0, 3646.749], [1048576.0, 4414.047]], "tps_vs_ctx": [[128.0, 6482.5], [155.0, 6438.3], [188.0, 6385.6], [228.0, 6322.9], [276.0, 6248.6], [334.0, 6160.9], [404.0, 6057.8], [490.0, 5937.6], [593.0, 5798.1], [719.0, 5637.7], [871.0, 5455.0], [1055.0, 5248.8], [1278.0, 5019.1], [1548.0, 4766.4], [1875.0, 4492.4], [2271.0, 4200.0], [2751.0, 3893.0], [3332.0, 3576.3], [4036.0, 3255.6], [4889.0, 2936.5], [5922.0, 2624.9], [7174.0, 2325.9], [8690.0, 2043.9], [10526.0, 1782.2], [12751.0, 1542.9], [15445.0, 1327.0], [18710.0, 1134.7], [22664.0, 965.3], [27453.0, 817.4], [33255.0, 689.5], [40283.0, 579.6], [48796.0, 485.8], [59108.0, 406.2], [71599.0, 338.9], [86731.0, 282.3], [105060.0, 234.8], [127262.0, 195.0], [154157.0, 161.8], [186736.0, 134.1], [226199.0, 111.1], [274003.0, 92.0], [331908.0, 76.1], [402052.0, 63.0], [487018.0, 52.1], [589942.0, 43.0], [714616.0, 35.6], [865638.0, 29.4], [1048576.0, 24.3]], "weight_gb": 16.0, "kv_total_gb": 34.36, "capacity_gb": 80.0, "footprint_gb": 50.36, "kv_bpt": 131072.0, "batch": 32, "context": 8192, "kv_frac": 0.6823, "fits": true, "bottleneck": "KV-cache-bound"}, "steps": 96, "node": "ip-172-26-4-148", "compute_ms": 0.2}