Data Parallelism (DDP): Multi-GPU LLM Training'in Temeli — AllReduce ve NCCL Anatomi

# DDP production-grade PyTorch
import os
import torch
import torch.nn as nn
import torch.optim as optim
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DataLoader, DistributedSampler
 
def setup():
    dist.init_process_group(backend='nccl')
    rank = dist.get_rank()
    world_size = dist.get_world_size()
    torch.cuda.set_device(rank)
    return rank, world_size
 
def cleanup():
    dist.destroy_process_group()
 
def train(model, train_dataset, num_epochs=10, batch_size=32):
    rank, world_size = setup()
    
    # Wrap model in DDP
    model = model.cuda(rank)
    model = DDP(model, device_ids=[rank])
    
    # Distributed sampler ensures each GPU sees different batch
    sampler = DistributedSampler(
        train_dataset,
        num_replicas=world_size,
        rank=rank,
        shuffle=True,
    )
    
    loader = DataLoader(
        train_dataset,
        batch_size=batch_size,
        sampler=sampler,
        num_workers=4,
        pin_memory=True,
    )
    
    optimizer = optim.AdamW(model.parameters(), lr=3e-4, betas=(0.9, 0.95))
    
    for epoch in range(num_epochs):
        sampler.set_epoch(epoch)  # Important for shuffling
        
        for step, (input_ids, labels) in enumerate(loader):
            input_ids = input_ids.cuda(rank)
            labels = labels.cuda(rank)
            
            optimizer.zero_grad()
            logits = model(input_ids)
            loss = nn.functional.cross_entropy(logits.view(-1, logits.size(-1)), labels.view(-1))
            loss.backward()    # AllReduce gradients here
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            optimizer.step()
            
            if step % 100 == 0 and rank == 0:
                print(f"Epoch {epoch} Step {step}: loss={loss.item():.4f}")
    
    cleanup()
 
if __name__ == '__main__':
    # Launch with: torchrun --nproc_per_node=4 train_ddp.py
    # MyModel + dataset definitions
    # train(MyModel(), MyDataset())
    pass
 
# Launch script
"""
# Single node, 4 GPUs:
torchrun --nproc_per_node=4 train_ddp.py
 
# Multi-node (2 nodes × 4 GPUs):
# Node 0:
torchrun --nproc_per_node=4 --nnodes=2 --node_rank=0 --master_addr=10.0.0.1 --master_port=29500 train_ddp.py
# Node 1:
torchrun --nproc_per_node=4 --nnodes=2 --node_rank=1 --master_addr=10.0.0.1 --master_port=29500 train_ddp.py
"""