DeepSeek-V3 / R1 (671B, 37B Active): Shared Expert + Fine-Grained Routing — LoRA Hangi Parçaya?

# === DeepSeek-V3 Expert Parallelism (EP) ===
# EP = expert'leri GPU'lar arasında dağıt
# 256 expert / 8 EP = 32 expert per GPU
 
# Megatron-DeepSpeed config
ep_size = 8                  # 256 experts / 8 = 32 per GPU
tp_size = 2                  # Tensor parallel for attention
pp_size = 1                  # No pipeline
dp_size = 16 / (ep_size * tp_size)  # Data parallel
 
# DeepSpeed config
{
  "zero_optimization": {
    "stage": 3,
    "offload_param": { "device": "none" },
    "offload_optimizer": { "device": "cpu" },     # 37B active params → optim CPU
  },
  "moe": {
    "enabled": true,
    "ep_size": 8,
    "experts_per_layer": 256,
    "load_balancing_type": "aux_free",            # DeepSeek-V3 stilinde
    "use_residual": true,                          # shared expert
  },
}