Profiling Stack: torch.profiler + Nsight Systems + Nsight Compute + MFU Calculation

# === MFU hesabı — cookbook helper ===
import torch, time
 
GPU_PEAK_TFLOPS = {
    "NVIDIA GeForce RTX 4090": 165,         # bf16 dense
    "NVIDIA A100-SXM4-80GB": 312,
    "NVIDIA H100 80GB HBM3": 989,
    "NVIDIA H100 PCIe": 756,
}
 
def measure_mfu(model, tokens_per_sec: float) -> float:
    n_params = sum(p.numel() for p in model.parameters())
    flops_per_token = 6 * n_params           # forward + backward
    achieved_tflops = tokens_per_sec * flops_per_token / 1e12
    gpu = torch.cuda.get_device_name(0)
    peak = GPU_PEAK_TFLOPS.get(gpu, None)
    if peak is None:
        print(f"⚠️ unknown GPU {gpu}; using A100 peak as fallback")
        peak = 312
    mfu = achieved_tflops / peak * 100
    print(f"params={n_params/1e9:.1f}B  achieved={achieved_tflops:.1f} TFLOPs/s  peak={peak} TFLOPs/s  →  MFU={mfu:.1f}%")
    return mfu
 
# Kullanım örneği — bench loop sonunda
# tokens_per_sec = total_tokens / elapsed_time
# measure_mfu(model, tokens_per_sec)

# === torch.profiler örneği ===
from torch.profiler import profile, ProfilerActivity, record_function
 
def train_step(model, batch):
    with record_function("forward"):
        out = model(**batch)
    with record_function("backward"):
        out.loss.backward()
    with record_function("optimizer_step"):
        optimizer.step()
        optimizer.zero_grad()
    return out.loss.item()
 
# Profile 10 step
with profile(
    activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],
    record_shapes=True,
    profile_memory=True,
    with_stack=True,
    schedule=torch.profiler.schedule(wait=2, warmup=2, active=6, repeat=1),
    on_trace_ready=torch.profiler.tensorboard_trace_handler("./tb_profiler"),
) as prof:
    for step, batch in enumerate(loader):
        if step >= 10: break
        loss = train_step(model, batch)
        prof.step()
 
# Top-10 CUDA op
print(prof.key_averages().table(
    sort_by="cuda_time_total", row_limit=10
))
 
# Sonra: tensorboard --logdir tb_profiler
# Browser'da operator breakdown, stack trace, memory timeline gör

# === nsys ile profile al ===
# RTX 4090 + cookbook standart komut
nsys profile \
    --output ftc_llama8b_qlora \
    --trace=cuda,nvtx,osrt,cudnn,cublas \
    --cuda-memory-usage true \
    --capture-range=cudaProfilerApi \
    --cudabacktrace=true \
    --gpu-metrics-device=0 \
    uv run python train.py --max_steps 20
 
# Trace yüklemek için: nsys-ui ftc_llama8b_qlora.nsys-rep
# Veya web UI: https://nsight-systems-web.nvidia.com (rapor yükle)

# === MFU benchmark — cookbook'un her Lab'ın sonunda zorunlu ===
import time, torch
from torch.utils.data import DataLoader
 
def bench_mfu(model, loader, n_steps: int = 50):
    """50 step'lik MFU bench. Cookbook sertifika gereksinimi."""
    model.train()
    optimizer = torch.optim.AdamW(model.parameters(), lr=2e-4)
 
    # Warmup
    for i, batch in enumerate(loader):
        if i >= 5: break
        batch = {k: v.to("cuda") for k, v in batch.items()}
        loss = model(**batch).loss
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()
    torch.cuda.synchronize()
 
    # Bench
    total_tokens = 0
    t0 = time.perf_counter()
    for i, batch in enumerate(loader):
        if i >= n_steps: break
        batch = {k: v.to("cuda") for k, v in batch.items()}
        total_tokens += batch["input_ids"].numel()
        loss = model(**batch).loss
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()
    torch.cuda.synchronize()
    elapsed = time.perf_counter() - t0
 
    tps = total_tokens / elapsed
    mfu = measure_mfu(model, tps)
    return {"tokens_per_sec": tps, "mfu_percent": mfu, "elapsed_sec": elapsed}
 
# Cookbook sertifika kontrol
result = bench_mfu(model, loader, n_steps=50)
assert result["mfu_percent"] >= 35, (
    f"MFU {result['mfu_percent']:.1f}% — cookbook minimum 35%. "
    f"Bottleneck'i tespit et: torch.profiler ile timeline çıkar."
)
print(f"✅ MFU {result['mfu_percent']:.1f}% — cookbook standardına uygun")