{"model": "reduced-order 3D-parallelism (TP x PP x DP) communication model -- compute roofline + TP/PP/DP collective costs, pipeline bubble, backward overlap (educational)", "inputs": {"model_params_b": 70.0, "num_gpus": 64, "tensor_parallel": 8, "pipeline_parallel": 4, "micro_batch": 1, "gradient_accum_steps": 16, "seq_length": 4096, "hidden_dim": 8192, "num_layers": 80, "link_bandwidth_gb_s": 450.0, "compute_tflops": 500.0}, "outputs": {"data_parallel": 2, "used_gpus": 64, "idle_gpus": 0, "valid_layout": true, "tokens_per_step": 131072, "compute_time_ms": 1720.32, "tp_comm_ms": 1336.212, "pp_comm_ms": 4.772, "pipeline_bubble_ms": 322.56, "dp_allreduce_ms": 0.0, "dp_allreduce_raw_ms": 9.722, "step_time_ms": 3383.864, "comm_fraction_percent": 39.6, "bubble_fraction_percent": 9.5, "scaling_efficiency_percent": 50.8, "mfu_percent": 50.8, "achieved_tflops_per_gpu": 254.2, "throughput_tokens_s": 38734.0, "per_gpu_tokens_s": 605.2, "bottleneck": "compute", "verdict": "overhead-limited -- compute is still the largest single term, but the combined communication + pipeline-bubble overhead drags model-FLOPs utilisation to 51%. Raise gradient-accumulation micro-batches to shrink the bubble, or widen the interconnect to hide the collectives"}, "profile": {"eff_vs_gpus": [[32, 50.84], [32, 50.84], [32, 50.84], [32, 50.84], [32, 50.84], [32, 50.84], [32, 50.84], [32, 50.84], [64, 50.84], [64, 50.84], [64, 50.84], [64, 50.84], [64, 50.84], [96, 50.84], [96, 50.84], [96, 50.84], [128, 50.84], [128, 50.84], [128, 50.84], [160, 50.84], [160, 50.84], [192, 50.84], [224, 50.84], [224, 50.84], [256, 50.84], [288, 50.84], [288, 50.84], [320, 50.84], [352, 50.84], [416, 50.84], [448, 50.84], [480, 50.84], [512, 50.84], [576, 50.84], [640, 50.84], [704, 50.84], [768, 50.84], [832, 50.84], [896, 50.84], [992, 50.84], [1088, 50.84], [1184, 50.84], [1312, 50.84], [1408, 50.84], [1568, 50.84], [1696, 50.84], [1856, 50.84], [2048, 50.84]], "eff_vs_bw": [[20.0, 5.34], [38.7, 9.76], [57.4, 13.71], [76.2, 17.26], [94.9, 20.48], [113.6, 23.39], [132.3, 26.05], [151.1, 28.49], [169.8, 30.74], [188.5, 32.81], [207.2, 34.72], [226.0, 36.5], [244.7, 38.15], [263.4, 39.7], [282.1, 41.14], [300.9, 42.49], [319.6, 43.76], [338.3, 44.96], [357.0, 46.08], [375.7, 47.15], [394.5, 48.15], [413.2, 49.11], [431.9, 50.01], [450.6, 50.87], [469.4, 51.68], [488.1, 52.46], [506.8, 53.2], [525.5, 53.91], [544.3, 54.59], [563.0, 55.23], [581.7, 55.85], [600.4, 56.44], [619.1, 57.01], [637.9, 57.56], [656.6, 58.08], [675.3, 58.59], [694.0, 59.07], [712.8, 59.54], [731.5, 59.99], [750.2, 60.42], [768.9, 60.84], [787.7, 61.24], [806.4, 61.63], [825.1, 62.01], [843.8, 62.38], [862.6, 62.73], [881.3, 63.07], [900.0, 63.4]], "compute_ms": 1720.32, "tp_ms": 1336.212, "pp_ms": 4.772, "bubble_ms": 322.56, "dp_ms": 0.0, "step_ms": 3383.864, "num_gpus": 64, "link_bandwidth_gb_s": 450.0, "mfu_percent": 50.8}, "steps": 96, "node": "ip-172-26-4-148", "compute_ms": 0.2}