{"model": "reduced-order FlashAttention model -- tiled online-softmax HBM traffic (linear vs quadratic) + roofline arithmetic intensity + SRAM tiling (educational)", "inputs": {"batch_size": 8, "num_heads": 32, "seq_length": 8192, "head_dim": 128, "elem_bytes": 2.0, "sram_kb": 192.0, "block_size": 128, "causal": 1, "hbm_bandwidth_gb_s": 3350.0, "compute_tops": 990.0}, "outputs": {"format": "b8 h32 S8192 d128 bf16 causal", "achieved_block": 128, "max_block_sram": 192, "sram_needed_kb": 128.0, "sram_used_kb": 128.0, "sram_budget_kb": 192.0, "sram_fit": true, "q_blocks": 64, "kv_blocks": 64, "flops_gflop": 4398.05, "scores_matrix_gb": 17.1799, "naive_hbm_gb": 36.5072, "fa_hbm_gb": 2.1475, "hbm_reduction_x": 17.0, "machine_balance_flop_byte": 295.5, "arithmetic_intensity_fa": 2048.0, "arithmetic_intensity_naive": 120.5, "fa_bottleneck": "compute", "naive_bottleneck": "memory bandwidth", "naive_time_ms": 10.898, "fa_time_ms": 4.442, "compute_time_ms": 4.442, "speedup_x": 2.45, "fa_tflops": 990.0, "fa_util_percent": 100.0, "verdict": "compute-bound win -- FlashAttention converts a memory-bound O(S^2) attention into a compute-bound kernel: HBM traffic drops 17.0x (the 17.2 GB scores matrix is never written), wall-clock is 2.5x faster, and the kernel now runs at 100% of peak tensor-core throughput. This is exactly the regime FlashAttention is built for"}, "profile": {"reduction_vs_seq": [[512.0, 2.0], [576.1, 2.125], [648.3, 2.266], [729.4, 2.425], [820.8, 2.603], [923.6, 2.804], [1039.2, 3.03], [1169.3, 3.284], [1315.8, 3.57], [1480.5, 3.892], [1665.9, 4.254], [1874.6, 4.661], [2109.3, 5.12], [2373.4, 5.636], [2670.7, 6.216], [3005.1, 6.869], [3381.4, 7.604], [3804.8, 8.431], [4281.3, 9.362], [4817.4, 10.409], [5420.7, 11.587], [6099.5, 12.913], [6863.3, 14.405], [7722.7, 16.083], [8689.8, 17.972], [9778.0, 20.098], [11002.4, 22.489], [12380.2, 25.18], [13930.5, 28.208], [15674.9, 31.615], [17637.8, 35.449], [19846.5, 39.763], [22331.7, 44.617], [25128.2, 50.079], [28274.9, 56.224], [31815.6, 63.14], [35799.7, 70.921], [40282.7, 79.677], [45327.1, 89.529], [51003.1, 100.615], [57390.0, 113.09], [64576.6, 127.126], [72663.1, 142.92], [81762.3, 160.692], [92001.0, 180.689], [103521.8, 203.191], [116485.2, 228.51], [131072.0, 257.0]], "speedup_vs_seq": [[512.0, 2.0], [576.1, 2.125], [648.3, 2.266], [729.4, 2.425], [820.8, 2.603], [923.6, 2.804], [1039.2, 3.03], [1169.3, 3.284], [1315.8, 3.207], [1480.5, 3.107], [1665.9, 3.018], [1874.6, 2.939], [2109.3, 2.869], [2373.4, 2.807], [2670.7, 2.751], [3005.1, 2.702], [3381.4, 2.658], [3804.8, 2.619], [4281.3, 2.585], [4817.4, 2.554], [5420.7, 2.527], [6099.5, 2.503], [6863.3, 2.481], [7722.7, 2.462], [8689.8, 2.445], [9778.0, 2.43], [11002.4, 2.416], [12380.2, 2.404], [13930.5, 2.394], [15674.9, 2.384], [17637.8, 2.376], [19846.5, 2.368], [22331.7, 2.362], [25128.2, 2.356], [28274.9, 2.351], [31815.6, 2.346], [35799.7, 2.342], [40282.7, 2.338], [45327.1, 2.335], [51003.1, 2.332], [57390.0, 2.329], [64576.6, 2.327], [72663.1, 2.325], [81762.3, 2.323], [92001.0, 2.322], [103521.8, 2.32], [116485.2, 2.319], [131072.0, 2.318]], "ai_fa": 2048.0, "ai_naive": 120.5, "machine_balance": 295.5, "peak_tflops": 990.0, "hbm_bw_gb_s": 3350.0, "hbm_reduction_x": 17.0, "seq_length": 8192, "speedup_x": 2.45}, "steps": 96, "node": "ip-172-26-4-148", "compute_ms": 0.2}