{"model": "reduced-order Mixture-of-Experts model -- top-k gating, capacity/drop (truncated-normal load) + padding waste + compute sparsity vs dense + expert-parallel all-to-all (educational)", "inputs": {"batch_size": 16, "seq_length": 2048, "num_experts": 64, "top_k": 2, "d_model": 4096, "d_ff": 14336, "capacity_factor": 1.25, "load_imbalance": 0.3, "expert_parallel": 8, "elem_bytes": 2.0, "network_bw_gb_s": 450.0, "compute_tops": 990.0}, "outputs": {"format": "T=32768 E64 top-2 d4096 ff14336 bf16 EP8", "tokens": 32768, "routed_assignments": 65536, "mean_load_per_expert": 1024.0, "capacity_per_expert": 1280.0, "dropped_tokens": 2228, "drop_percent": 3.4, "padding_waste_percent": 22.72, "slot_utilization_percent": 77.28, "total_expert_params_b": 7.516, "active_params_b": 0.235, "sparsity_percent": 3.12, "expert_weights_gb": 15.032, "moe_tflop": 19.241, "dense_equiv_tflop": 492.581, "compute_savings_x": 25.6, "all2all_gb": 1.1744, "compute_time_ms": 2.429, "all2all_time_ms": 0.326, "step_time_ms": 2.756, "comm_fraction_percent": 11.8, "bottleneck": "expert compute", "achieved_tflops": 674.5, "mfu_percent": 68.1, "verdict": "balanced-efficient -- top-2 of 64 experts fire, so only 3.1% of the parameters are active and the layer does 25.6x less matmul than a dense FFN of the same 15.0 GB of total weights. Drops are 3.4%, only 23% of slots padded, and the layer is expert compute-bound at 68% MFU -- a healthy MoE operating point"}, "profile": {"drop_vs_capacity": [[0.5, 50.595], [0.574, 43.609], [0.649, 36.895], [0.723, 30.553], [0.798, 24.693], [0.872, 19.419], [0.947, 14.815], [1.021, 10.935], [1.096, 7.785], [1.17, 5.334], [1.245, 3.508], [1.319, 2.21], [1.394, 1.331], [1.468, 0.765], [1.543, 0.419], [1.617, 0.219], [1.691, 0.108], [1.766, 0.051], [1.84, 0.023], [1.915, 0.01], [1.989, 0.004], [2.064, 0.001], [2.138, 0.001], [2.213, 0.0], [2.287, 0.0], [2.362, 0.0], [2.436, 0.0], [2.511, 0.0], [2.585, 0.0], [2.66, 0.0], [2.734, 0.0], [2.809, 0.0], [2.883, 0.0], [2.957, 0.0], [3.032, 0.0], [3.106, 0.0], [3.181, 0.0], [3.255, 0.0], [3.33, 0.0], [3.404, 0.0], [3.479, 0.0], [3.553, 0.0], [3.628, 0.0], [3.702, 0.0], [3.777, 0.0], [3.851, 0.0], [3.926, 0.0], [4.0, 0.0]], "waste_vs_capacity": [[0.5, 1.19], [0.574, 1.838], [0.649, 2.756], [0.723, 4.0], [0.798, 5.615], [0.872, 7.626], [0.947, 10.03], [1.021, 12.79], [1.096, 15.843], [1.17, 19.104], [1.245, 22.477], [1.319, 25.869], [1.394, 29.199], [1.468, 32.405], [1.543, 35.444], [1.617, 38.293], [1.691, 40.945], [1.766, 43.402], [1.84, 45.677], [1.915, 47.783], [1.989, 49.735], [2.064, 51.547], [2.138, 53.234], [2.213, 54.808], [2.287, 56.279], [2.362, 57.658], [2.436, 58.952], [2.511, 60.169], [2.585, 61.317], [2.66, 62.4], [2.734, 63.424], [2.809, 64.394], [2.883, 65.314], [2.957, 66.187], [3.032, 67.018], [3.106, 67.808], [3.181, 68.562], [3.255, 69.281], [3.33, 69.968], [3.404, 70.625], [3.479, 71.254], [3.553, 71.856], [3.628, 72.434], [3.702, 72.989], [3.777, 73.521], [3.851, 74.033], [3.926, 74.526], [4.0, 75.0]], "expert_loads": [1.6726, 1.5343, 1.4603, 1.407, 1.364, 1.3275, 1.2953, 1.2661, 1.2393, 1.2142, 1.1906, 1.1681, 1.1465, 1.1257, 1.1054, 1.0856, 1.0662, 1.0471, 1.0282, 1.0094, 0.9906, 0.9718, 0.9529, 0.9338, 0.9144, 0.8946, 0.8743, 0.8535, 0.8319, 0.8094, 0.7858, 0.7607, 0.7339, 0.7047, 0.6725, 0.636, 0.593, 0.5397, 0.4657, 0.3274], "capacity_norm": 1.25, "num_experts": 64, "top_k": 2, "sparsity_percent": 3.12, "compute_savings_x": 25.6, "drop_percent": 3.4, "padding_waste_percent": 22.72, "comm_fraction_percent": 11.8, "capacity_factor": 1.25}, "steps": 96, "node": "ip-172-26-4-148", "compute_ms": 1.1}