{"model": "reduced-order LLM training-memory model -- model states, sharding, activations, capacity fit (educational)", "inputs": {"model_params_b": 8.0, "num_gpus": 8, "gpu_memory_gb": 80.0, "micro_batch": 4, "seq_length": 4096, "num_layers": 32, "hidden_dim": 4096, "precision_bits": 16.0, "zero_stage": 3, "grad_ckpt": 1}, "outputs": {"model_states_unsharded_gb": 128.0, "model_states_per_gpu_gb": 16.0, "activations_per_gpu_gb": 8.858, "per_gpu_total_gb": 24.858, "gpu_capacity_gb": 80.0, "utilization_percent": 31.1, "fits": true, "bytes_per_param": 16.0, "weights_per_gpu_gb": 2.0, "grads_per_gpu_gb": 2.0, "optimizer_per_gpu_gb": 12.0, "min_gpus_to_fit": 2, "aggregate_cluster_gb": 198.9, "global_batch": 32, "zero_stage": 3, "checkpointing": true, "verdict": "model-state-dominated -- optimizer states and weights dominate; sharding across more GPUs (higher ZeRO/FSDP stage) is what frees memory here, not checkpointing; ZeRO-3 / FSDP (fully sharded)"}, "profile": {"mem_vs_gpus": [[1, 136.858], [1, 136.858], [1, 136.858], [2, 72.858], [2, 72.858], [2, 72.858], [3, 51.525], [3, 51.525], [4, 40.858], [4, 40.858], [5, 34.458], [6, 30.192], [7, 27.144], [8, 24.858], [10, 21.658], [11, 20.495], [13, 18.705], [16, 16.858], [19, 15.595], [22, 14.677], [26, 13.781], [30, 13.125], [35, 12.516], [42, 11.906], [49, 11.471], [58, 11.065], [68, 10.741], [80, 10.458], [94, 10.22], [110, 10.022], [130, 9.843], [153, 9.695], [180, 9.569], [211, 9.465], [249, 9.372], [292, 9.297], [344, 9.23], [404, 9.175], [476, 9.127], [559, 9.087], [658, 9.053], [774, 9.024], [910, 8.999], [1070, 8.978], [1259, 8.96], [1481, 8.945], [1741, 8.932], [2048, 8.921]], "mem_vs_seq": [[512.0, 17.107], [576.0, 17.246], [648.0, 17.402], [729.0, 17.578], [821.0, 17.775], [924.0, 17.997], [1039.0, 18.247], [1169.0, 18.529], [1316.0, 18.846], [1481.0, 19.202], [1666.0, 19.603], [1875.0, 20.054], [2109.0, 20.562], [2373.0, 21.133], [2671.0, 21.776], [3005.0, 22.499], [3381.0, 23.313], [3805.0, 24.229], [4281.0, 25.259], [4817.0, 26.419], [5421.0, 27.723], [6099.0, 29.191], [6863.0, 30.843], [7723.0, 32.702], [8690.0, 34.793], [9778.0, 37.147], [11002.0, 39.795], [12380.0, 42.774], [13930.0, 46.127], [15675.0, 49.9], [17638.0, 54.145], [19846.0, 58.922], [22332.0, 64.297], [25128.0, 70.345], [28275.0, 77.15], [31816.0, 84.807], [35800.0, 93.424], [40283.0, 103.119], [45327.0, 114.028], [51003.0, 126.304], [57390.0, 140.117], [64577.0, 155.659], [72663.0, 173.148], [81762.0, 192.826], [92001.0, 214.969], [103522.0, 239.885], [116485.0, 267.921], [131072.0, 299.468]], "model_state_gb": 16.0, "activation_gb": 8.858, "capacity_gb": 80.0, "per_gpu_gb": 24.858, "num_gpus": 8, "seq_length": 4096, "fits": true}, "steps": 96, "node": "ip-172-26-4-148", "compute_ms": 0.9}